From a07f129d97493559acb80dd691187698ef54718b Mon Sep 17 00:00:00 2001 From: "A. Alonso Dominguez" <2269440+alonsodomin@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:45:29 +0200 Subject: [PATCH] Proposal for a Helm Deployment goal implementation (#15882) Implementation for the proposal of a Helm Deployment goal. Design document can be found in: https://docs.google.com/document/d/17wZRjiIxM5918ybC_0EnKY42qwd47-1ZporBdua8vRQ/edit?usp=sharing --- docs/markdown/Helm/helm-deployments.md | 208 ++++ docs/markdown/Helm/helm-overview.md | 2 +- pants.toml | 6 +- src/python/pants/backend/docker/register.py | 2 + .../pants/backend/docker/target_types.py | 18 + .../backend/experimental/helm/register.py | 15 +- .../helm/dependency_inference/chart.py | 2 + .../helm/dependency_inference/deployment.py | 186 +++ .../dependency_inference/deployment_test.py | 224 ++++ .../dependency_inference/unittest_test.py | 8 +- src/python/pants/backend/helm/goals/deploy.py | 89 ++ .../pants/backend/helm/goals/deploy_test.py | 162 +++ src/python/pants/backend/helm/goals/lint.py | 7 +- .../pants/backend/helm/goals/lint_test.py | 26 +- .../pants/backend/helm/goals/package.py | 10 +- .../pants/backend/helm/goals/package_test.py | 8 +- .../pants/backend/helm/goals/publish.py | 15 +- .../pants/backend/helm/resolve/fetch.py | 3 +- .../pants/backend/helm/subsystems/BUILD | 40 +- .../pants/backend/helm/subsystems/helm.py | 95 +- .../backend/helm/subsystems/k8s_parser.lock | 1023 +++++++++++++++++ .../backend/helm/subsystems/k8s_parser.py | 151 +++ .../helm/subsystems/k8s_parser_main.py | 36 + .../helm/subsystems/k8s_parser_test.py | 76 ++ .../helm/subsystems/post_renderer.lock | 134 +++ .../backend/helm/subsystems/post_renderer.py | 224 ++++ .../helm/subsystems/post_renderer_main.py | 127 ++ .../helm/subsystems/post_renderer_test.py | 76 ++ .../pants/backend/helm/subsystems/unittest.py | 4 +- .../backend/helm/subsystems/unittest_test.py | 8 +- src/python/pants/backend/helm/target_types.py | 108 +- .../pants/backend/helm/target_types_test.py | 4 +- .../pants/backend/helm/test/unittest.py | 5 +- .../pants/backend/helm/test/unittest_test.py | 13 +- src/python/pants/backend/helm/testutil.py | 84 +- .../pants/backend/helm/util_rules/chart.py | 107 +- .../backend/helm/util_rules/chart_metadata.py | 2 +- .../backend/helm/util_rules/chart_test.py | 108 +- .../pants/backend/helm/util_rules/plugins.py | 180 --- .../backend/helm/util_rules/post_renderer.py | 134 +++ .../helm/util_rules/post_renderer_test.py | 199 ++++ .../pants/backend/helm/util_rules/renderer.py | 406 +++++++ .../backend/helm/util_rules/renderer_test.py | 194 ++++ .../pants/backend/helm/util_rules/sources.py | 3 +- .../backend/helm/util_rules/sources_test.py | 8 +- .../pants/backend/helm/util_rules/tool.py | 278 ++++- .../backend/helm/util_rules/tool_test.py | 10 +- .../backend/helm/util_rules/yaml_utils.py | 29 - src/python/pants/backend/helm/utils/BUILD | 6 + .../pants/backend/helm/utils/__init__.py | 0 src/python/pants/backend/helm/utils/yaml.py | 273 +++++ .../pants/backend/helm/utils/yaml_test.py | 46 + .../pants/core/util_rules/system_binaries.py | 22 +- 53 files changed, 4836 insertions(+), 368 deletions(-) create mode 100644 docs/markdown/Helm/helm-deployments.md create mode 100644 src/python/pants/backend/helm/dependency_inference/deployment.py create mode 100644 src/python/pants/backend/helm/dependency_inference/deployment_test.py create mode 100644 src/python/pants/backend/helm/goals/deploy.py create mode 100644 src/python/pants/backend/helm/goals/deploy_test.py create mode 100644 src/python/pants/backend/helm/subsystems/k8s_parser.lock create mode 100644 src/python/pants/backend/helm/subsystems/k8s_parser.py create mode 100644 src/python/pants/backend/helm/subsystems/k8s_parser_main.py create mode 100644 src/python/pants/backend/helm/subsystems/k8s_parser_test.py create mode 100644 src/python/pants/backend/helm/subsystems/post_renderer.lock create mode 100644 src/python/pants/backend/helm/subsystems/post_renderer.py create mode 100644 src/python/pants/backend/helm/subsystems/post_renderer_main.py create mode 100644 src/python/pants/backend/helm/subsystems/post_renderer_test.py delete mode 100644 src/python/pants/backend/helm/util_rules/plugins.py create mode 100644 src/python/pants/backend/helm/util_rules/post_renderer.py create mode 100644 src/python/pants/backend/helm/util_rules/post_renderer_test.py create mode 100644 src/python/pants/backend/helm/util_rules/renderer.py create mode 100644 src/python/pants/backend/helm/util_rules/renderer_test.py delete mode 100644 src/python/pants/backend/helm/util_rules/yaml_utils.py create mode 100644 src/python/pants/backend/helm/utils/BUILD create mode 100644 src/python/pants/backend/helm/utils/__init__.py create mode 100644 src/python/pants/backend/helm/utils/yaml.py create mode 100644 src/python/pants/backend/helm/utils/yaml_test.py diff --git a/docs/markdown/Helm/helm-deployments.md b/docs/markdown/Helm/helm-deployments.md new file mode 100644 index 000000000000..f02571558610 --- /dev/null +++ b/docs/markdown/Helm/helm-deployments.md @@ -0,0 +1,208 @@ +--- +title: "Deployments" +slug: "helm-deployments" +hidden: false +createdAt: "2022-07-19T13:16:00.000Z" +updatedAt: "2022-07-19T13:16:00.000Z" +--- +> 🚧 Helm deployment support is in alpha stage +> +> Pants has experimental support for managing deployments via the `experimental-deploy` goal. Helm deployments provides with a basic implementation of this goal. +> +> Please share feedback for what you need to use Pants with your Helm deployments by either [opening a GitHub issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our Slack](doc:getting-help)! + +Motivation +---------- + +Helm's ultimate purpose is to simplify the deployment of Kubernetes resources and help in making these reproducible. However it is quite common to deploy the same software application into different kind of environments using slightly different configuration overrides. + +This hinders reproducibility since operators end up having a set of configuration files and additional shell scripts that ensure that the Helm command line usued to deploy a piece of software into a given environment is always the same. + +Pants solves this problem by providing with the ability to manage the configuration files and the different parameters of a deployment as single unit such that a simple command line as `./pants experimental-deploy ::` will always have the same effect on each of the deployments previously defined. + +Defining Helm deployments +------------------------- + +Helm deployments are defined using the `helm_deployment` target which has a series of fields that can be used to guarantee the reproducibility of the given deployment. `helm_deployment` targets need to be added by hand as there is no deterministic way of instrospecting your repository to find sources that are specific to Helm: + +```text src/chart/BUILD +helm_chart() +``` +```yaml src/chart/Chart.yaml +apiVersion: v2 +description: Example Helm chart +name: example +version: 0.1.0 +``` +```text src/deployment/BUILD +helm_deployment(name="dev", sources=["common-values.yaml", "dev-override.yaml"], dependencies=["//src/chart"]) + +helm_deployment(name="stage", sources=["common-values.yaml", "stage-override.yaml"], dependencies=["//src/chart"]) + +helm_deployment(name="prod", sources=["common-values.yaml", "prod-override.yaml"], dependencies=["//src/chart"]) +``` +```yaml src/deployment/common-values.yaml +# Default values common to all deployments +env: + SERVICE_NAME: my-service +``` +```yaml src/deployment/dev-override.yaml +# Specific values to the DEV environment +env: + ENV_ID: dev +``` +```yaml src/deployment/stage-override.yaml +# Specific values to the STAGE environment +env: + ENV_ID: stage +``` +```yaml src/deployment/prod-override.yaml +# Specific values to the PRODUCTION environment +env: + ENV_ID: prod +``` + +There are quite a few things to notice in the previous example: + +* The `helm_deployment` target requires you to explicitly define as a dependency which chart to use. +* We have three different deployments that using configuration files with the specified chart. +* One of those value files (`common-values.yaml`) provides with default values that are common to all deployments. +* Each deployment uses an additional `xxx-override.yaml` file with values that are specific to the given deployment. + +> 📘 Source roots +> +> Don't forget to configure your source roots such that each of the shown files in the previous example sit at their respective source root level. + +The `helm_deployment` target has many additional fields including the target kubernetes namespace, adding inline override values (similar to using helm's `--set` arg) and many others. Please run `./pants help helm_deployment` to see all the posibilities. + +Dependencies with `docker_image` targets +---------------------------------------- + +A Helm deployment will in most cases deploy one or more Docker images into Kubernetes. Furthermore, it's quite likely there is going to be at least a few first party Docker images among those. Pants is capable of analysing the Helm chart being used in a deployment to detect those required first-party Docker images using Pants' target addresses to those Docker images. + +To illustrate this, let's imagine the following scenario: Let's say we have a first-party Docker image that we want to deploy into Kubernetes as a `Pod` resource kind. For achieving this we define the following workspace: + +```text src/docker/BUILD +docker_image() +``` +```text src/docker/Dockerfile +FROM busybox:1.28 +``` +```text src/chart/BUILD +helm_chart() +``` +```yaml src/chart/Chart.yaml +apiVersion: v2 +description: Example Helm chart +name: example +version: 0.1.0 +``` +```yaml src/chart/values.yaml +# Default image in case this chart is used by other tools after being published +image: example.com/registry/my-app:latest +``` +```yaml src/chart/templates/pod.yaml +--- +apiVersion: v1 +kind: Pod +metadata: + name: my_pod + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + containers: + - name: my_app + # Uses the `image` value entry from the deployment inputs + image: {{ .Values.image }} +``` +```text src/deployment/BUILD +# Overrides the `image` value for the chart using the target address for the first-party docker image. +helm_deployment(dependencies=["src/chart"], values={"image": "src/docker"}) +``` + +> 📘 Docker image references VS Pants' target addresses +> +> You should use typical Docker registry addresses in your Helm charts. Because Helm charts are distributable artifacts and may be used with tools other than Pants, you should create your charts such that when that chart is being used, all Docker image addresses are valid references to images in accessible Docker registries. As shown in the example, we recommend that you make the image address value configurable, especially for charts that deploy first-party Docker images. +> Your chart resources can still use off-the-shelf images published with other means, and in those cases you will also be referencing the Docker image address. Usage of Pants' target addresses is intended for your own first-party images because the image reference of those is not known at the time we create the sources (they are computed later). + +With this setup we should be able to run `./pants dependencies src/deployment` and Pants should give the following output: + +```text +src/chart +src/docker +``` + +This should work with any kind of Kubernetes resource that leads to Docker image being deployed into Kubernetes, such as `Deployment`, `StatefulSet`, `ReplicaSet`, `CronJob`, etc. Please get in touch with us in case you find Pants was not capable to infer dependencies in any of your `helm_deployment` targets by either [opening a GitHub issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our Slack](doc:getting-help). + +> 📘 How the Docker image reference is calculated during deployment? +> +> Pants' will rely on the behaviour of the `docker_image` target when it comes down to generate the final image reference. Since a given image may have more than one valid image reference, **Pants will try to use the first one that is not tagged as `latest`**, falling back to `latest` if none could be found. +> It's good practice to publish your Docker images using tags other than `latest` and Pants preferred behaviour is to choose those as this guarantees that the _version_ of the Docker image being deployed is the expected one. + +Value files +----------- + +It's very common that Helm deployments use a series of files providing with values that customise the given chart. When using deployments that may have more than one YAML file as the source of configuration values, the Helm backend needs to sort the file names in a way that is consistent across different machines, as the order in which those files are passed to the Helm command is relevant. The final order depends on the same order in which those files are specified in the `sources` field of the `helm_deployment` target. For example, given the following `BUILD` file: + +```text src/deployment/BUILD +helm_deployment(name="dev", dependencies=["//src/chart"], sources=["first.yaml", "second.yaml", "last.yaml"]) +``` + +This will result in the Helm command receiving the value files as in that exact order. + +If using any glob pattern in the `sources` field, the plugin will first group the files according to the order in which those glob patterns are listed. In this grouping, files that are resolved by more than one pattern will be part of the most specific group. Then we use alphanumeric ordering for the files that correspond to each of the previous groups. To illustrate this scenario, consider the following list of files: + +``` +src/deployment/002-config_maps.yaml +src/deployment/001-services.yaml +src/deployment/first.yaml +src/deployment/dev/daemon_sets.yaml +src/deployment/dev/services-override.yaml +src/deployment/last.yaml +``` + +And also the following `helm_deployment` target definition: + +```text src/deployment/BUILD +helm_deployment(name="dev", dependencies=["//src/chart"], sources=["first.yaml", "*.yaml", "dev/*-override.yaml", "dev/*.yaml", "last.yaml"]) +``` + +In this case, the final ordering of the files would be as follows: + +``` +src/deployment/first.yaml +src/deployment/001-services.yaml +src/deployment/002-config_maps.yaml +src/deployment/dev/services-override.yaml +src/deployment/dev/daemon_sets.yaml +src/deployment/last.yaml +``` + +We believe that this approach gives a very consistent and predictable ordering while at the same time total flexibility to the end user to organise their files as they best fit each particular case of a deployment. + +In addition to value files, you can also use inline values in your `helm_deployment` targets by means of the `values` field. All inlines values that are set this way will override any entry that may come from value files. + +Deploying +--------- + +Continuing with the example in the previous section, we can deploy it into Kubernetes using the command `./pants experimental-deploy src/deployment`. This will trigger the following steps: + +1. Analyse the dependencies of the given deployment. +2. Build and publish any first-party Docker image and Helm charts that are part of those dependencies. +3. Post-process the Kubernetes manifests generated by Helm by replacing all references to first-party Docker images by their real final registry destination. +4. Initiate the deployment of the final Kubernetes resources resulting from the post-processing. + +The `experimental-deploy` goal also supports default Helm pass-through arguments that allow to change the deployment behaviour to be either atomic or a dry-run or even what is the Kubernetes config file (the `kubeconfig` file) and target context to be used in the deployment. + +Please note that the list of valid pass-through arguments has been limited to those that do not alter the reproducibility of the deployment (i.e. `--create-namespace` is not a valid pass-through argument). Those arguments will have equivalent fields in the `helm_deployment` target. + +For example, to make an atomic deployment into a non-default Kubernetes context you can use a command like the following one: + +``` +./pants experimental-deploy src/deployments:prod -- --kube-context my-custom-kube-context --atomic +``` + +> 📘 How does Pants authenticate with the Kubernetes cluster? +> +> Short answer is: it doesn't. +> Pants will invoke Helm under the hood with the appropriate arguments to only perform the deployment. Any authentication steps that may be needed to perform the given deployment have to be done before invoking the `experimental-deploy` goal. If you are planning to run the deployment procedure from your CI/CD pipelines, ensure that all necessary preliminary steps (including authentication with the cluster) are done before the one that triggers the deployment. \ No newline at end of file diff --git a/docs/markdown/Helm/helm-overview.md b/docs/markdown/Helm/helm-overview.md index 0b77bcb75dee..b443dc486d69 100644 --- a/docs/markdown/Helm/helm-overview.md +++ b/docs/markdown/Helm/helm-overview.md @@ -53,7 +53,7 @@ Adding `helm_chart` targets Helm charts are identified by the presence of a `Chart.yaml` or `Chart.yml` file, which contains relevant metadata about the chart like its name, version, dependencies, etc. To get started quickly you can create a simple `Chart.yaml` file in your sources folder: -```text Chart.yaml +```yaml Chart.yaml apiVersion: v2 description: Example Helm chart name: example diff --git a/pants.toml b/pants.toml index 5ce3630c0ca5..81976d954f6e 100644 --- a/pants.toml +++ b/pants.toml @@ -119,9 +119,13 @@ venv_use_symlinks = true # + src/python/pants/testutil:testutil_wheel interpreter_constraints = [">=3.7,<3.10"] macos_big_sur_compatibility = true -resolves = { python-default = "3rdparty/python/user_reqs.lock" } enable_resolves = true +[python.resolves] +python-default = "3rdparty/python/user_reqs.lock" +helm-post-renderer = "src/python/pants/backend/helm/subsystems/post_renderer.lock" +helm-k8s-parser = "src/python/pants/backend/helm/subsystems/k8s_parser.lock" + [python-infer] assets = true unowned_dependency_behavior = "error" diff --git a/src/python/pants/backend/docker/register.py b/src/python/pants/backend/docker/register.py index 211f5d46b351..82cdf8e61c95 100644 --- a/src/python/pants/backend/docker/register.py +++ b/src/python/pants/backend/docker/register.py @@ -5,6 +5,7 @@ from pants.backend.docker.goals.tailor import rules as tailor_rules from pants.backend.docker.rules import rules as docker_rules from pants.backend.docker.target_types import DockerImageTarget +from pants.backend.docker.target_types import rules as target_types_rules def rules(): @@ -12,6 +13,7 @@ def rules(): *docker_rules(), *export_codegen_goal.rules(), *tailor_rules(), + *target_types_rules(), ) diff --git a/src/python/pants/backend/docker/target_types.py b/src/python/pants/backend/docker/target_types.py index 67c8b4005ed2..5a8edce95630 100644 --- a/src/python/pants/backend/docker/target_types.py +++ b/src/python/pants/backend/docker/target_types.py @@ -15,8 +15,10 @@ from pants.core.goals.run import RestartableField from pants.engine.addresses import Address from pants.engine.fs import GlobMatchErrorBehavior +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, + AllTargets, AsyncFieldMixin, BoolField, Dependencies, @@ -26,6 +28,7 @@ StringField, StringSequenceField, Target, + Targets, ) from pants.util.docutil import bin_name, doc_url from pants.util.strutil import softwrap @@ -369,3 +372,18 @@ class DockerImageTarget(Target): """ ) + + +class AllDockerImageTargets(Targets): + pass + + +@rule +def all_docker_targets(all_targets: AllTargets) -> AllDockerImageTargets: + return AllDockerImageTargets( + [tgt for tgt in all_targets if tgt.has_field(DockerImageSourceField)] + ) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/experimental/helm/register.py b/src/python/pants/backend/experimental/helm/register.py index 1ff6cdd686dc..159da0e2883e 100644 --- a/src/python/pants/backend/experimental/helm/register.py +++ b/src/python/pants/backend/experimental/helm/register.py @@ -1,22 +1,24 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.helm.goals import lint, package, publish, tailor +from pants.backend.helm.dependency_inference import deployment +from pants.backend.helm.goals import deploy, lint, package, publish, tailor from pants.backend.helm.target_types import ( HelmArtifactTarget, HelmChartTarget, + HelmDeploymentTarget, HelmUnitTestTestsGeneratorTarget, HelmUnitTestTestTarget, ) from pants.backend.helm.target_types import rules as target_types_rules -from pants.backend.helm.test.unittest import rules as test_rules -from pants.backend.helm.util_rules import chart, sources, tool +from pants.backend.helm.test.unittest import rules as unittest_rules def target_types(): return [ HelmArtifactTarget, HelmChartTarget, + HelmDeploymentTarget, HelmUnitTestTestTarget, HelmUnitTestTestsGeneratorTarget, ] @@ -24,13 +26,12 @@ def target_types(): def rules(): return [ - *chart.rules(), *lint.rules(), + *deploy.rules(), + *deployment.rules(), *package.rules(), *publish.rules(), *tailor.rules(), - *test_rules(), - *sources.rules(), - *tool.rules(), + *unittest_rules(), *target_types_rules(), ] diff --git a/src/python/pants/backend/helm/dependency_inference/chart.py b/src/python/pants/backend/helm/dependency_inference/chart.py index 072c84295bd7..23a8a85468fc 100644 --- a/src/python/pants/backend/helm/dependency_inference/chart.py +++ b/src/python/pants/backend/helm/dependency_inference/chart.py @@ -16,6 +16,7 @@ HelmChartMetaSourceField, HelmChartTarget, ) +from pants.backend.helm.target_types import rules as helm_target_types_rules from pants.backend.helm.util_rules.chart_metadata import HelmChartDependency, HelmChartMetadata from pants.engine.addresses import Address from pants.engine.internals.selectors import Get, MultiGet @@ -160,5 +161,6 @@ def rules(): return [ *collect_rules(), *artifacts.rules(), + *helm_target_types_rules(), UnionRule(InferDependenciesRequest, InferHelmChartDependenciesRequest), ] diff --git a/src/python/pants/backend/helm/dependency_inference/deployment.py b/src/python/pants/backend/helm/dependency_inference/deployment.py new file mode 100644 index 000000000000..9ffa53a5e919 --- /dev/null +++ b/src/python/pants/backend/helm/dependency_inference/deployment.py @@ -0,0 +1,186 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import PurePath + +from pants.backend.docker.target_types import AllDockerImageTargets +from pants.backend.docker.target_types import rules as docker_target_types_rules +from pants.backend.helm.subsystems import k8s_parser +from pants.backend.helm.subsystems.k8s_parser import ParsedKubeManifest, ParseKubeManifestRequest +from pants.backend.helm.target_types import ( + AllHelmDeploymentTargets, + HelmDeploymentDependenciesField, + HelmDeploymentFieldSet, +) +from pants.backend.helm.target_types import rules as helm_target_types_rules +from pants.backend.helm.util_rules import renderer +from pants.backend.helm.util_rules.renderer import ( + HelmDeploymentCmd, + HelmDeploymentRequest, + RenderedHelmFiles, +) +from pants.backend.helm.utils.yaml import FrozenYamlIndex, MutableYamlIndex +from pants.engine.addresses import Address +from pants.engine.fs import Digest, DigestEntries, FileEntry +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import ( + DependenciesRequest, + ExplicitlyProvidedDependencies, + FieldSet, + InferDependenciesRequest, + InferredDependencies, +) +from pants.engine.unions import UnionRule +from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel +from pants.util.ordered_set import FrozenOrderedSet, OrderedSet +from pants.util.strutil import pluralize, softwrap + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class HelmDeploymentReport: + address: Address + image_refs: FrozenYamlIndex[str] + + @property + def all_image_refs(self) -> FrozenOrderedSet[str]: + return FrozenOrderedSet(self.image_refs.values()) + + +@rule(desc="Analyse Helm deployment", level=LogLevel.DEBUG) +async def analyse_deployment(field_set: HelmDeploymentFieldSet) -> HelmDeploymentReport: + rendered_deployment = await Get( + RenderedHelmFiles, + HelmDeploymentRequest( + cmd=HelmDeploymentCmd.RENDER, + field_set=field_set, + description=f"Rendering Helm deployment {field_set.address}", + ), + ) + + rendered_entries = await Get(DigestEntries, Digest, rendered_deployment.snapshot.digest) + parsed_manifests = await MultiGet( + Get( + ParsedKubeManifest, + ParseKubeManifestRequest(file=entry), + ) + for entry in rendered_entries + if isinstance(entry, FileEntry) + ) + + # Build YAML index of Docker image refs for future processing during depedendecy inference or post-rendering. + image_refs_index: MutableYamlIndex[str] = MutableYamlIndex() + for manifest in parsed_manifests: + for (idx, path, image_ref) in manifest.found_image_refs: + image_refs_index.insert( + file_path=PurePath(manifest.filename), + document_index=idx, + yaml_path=path, + item=image_ref, + ) + + return HelmDeploymentReport(address=field_set.address, image_refs=image_refs_index.frozen()) + + +@dataclass(frozen=True) +class FirstPartyHelmDeploymentMappings: + """A mapping between `helm_deployment` target addresses and tuples made up of a Docker image + reference and a `docker_image` target address. + + The tuples of Docker image references and addresses are stored in a YAML index so we can track + the locations in which the Docker image refs appear in the deployment files. + """ + + deployment_to_docker_addresses: FrozenDict[Address, FrozenYamlIndex[tuple[str, Address]]] + + def docker_addresses_referenced_by(self, address: Address) -> list[tuple[str, Address]]: + if address not in self.deployment_to_docker_addresses: + return [] + return list(self.deployment_to_docker_addresses[address].values()) + + +@rule +async def first_party_helm_deployment_mappings( + deployment_targets: AllHelmDeploymentTargets, docker_targets: AllDockerImageTargets +) -> FirstPartyHelmDeploymentMappings: + field_sets = [HelmDeploymentFieldSet.create(tgt) for tgt in deployment_targets] + all_deployments_info = await MultiGet( + Get(HelmDeploymentReport, HelmDeploymentFieldSet, field_set) for field_set in field_sets + ) + + docker_target_addresses = {tgt.address.spec: tgt.address for tgt in docker_targets} + + def lookup_docker_addreses(image_ref: str) -> tuple[str, Address] | None: + addr = docker_target_addresses.get(str(image_ref), None) + if addr: + return image_ref, addr + return None + + # Builds a mapping between `helm_deployment` addresses and a YAML index of `docker_image` addresses. + address_mapping = { + fs.address: info.image_refs.transform_values(lookup_docker_addreses) + for fs, info in zip(field_sets, all_deployments_info) + } + return FirstPartyHelmDeploymentMappings( + deployment_to_docker_addresses=FrozenDict(address_mapping) + ) + + +@dataclass(frozen=True) +class HelmDeploymentDependenciesInferenceFieldSet(FieldSet): + required_fields = (HelmDeploymentDependenciesField,) + + dependencies: HelmDeploymentDependenciesField + + +class InferHelmDeploymentDependenciesRequest(InferDependenciesRequest): + infer_from = HelmDeploymentDependenciesInferenceFieldSet + + +@rule(desc="Find the dependencies needed by a Helm deployment") +async def inject_deployment_dependencies( + request: InferHelmDeploymentDependenciesRequest, mapping: FirstPartyHelmDeploymentMappings +) -> InferredDependencies: + explicitly_provided_deps = await Get( + ExplicitlyProvidedDependencies, DependenciesRequest(request.field_set.dependencies) + ) + candidate_docker_addresses = mapping.docker_addresses_referenced_by(request.field_set.address) + + dependencies: OrderedSet[Address] = OrderedSet() + for imager_ref, candidate_address in candidate_docker_addresses: + matches = frozenset([candidate_address]).difference(explicitly_provided_deps.includes) + explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( + matches, + request.field_set.address, + context=softwrap( + f"The Helm deployment {request.field_set.address} declares " + f"{imager_ref} as Docker image reference" + ), + import_reference="manifest", + ) + + maybe_disambiguated = explicitly_provided_deps.disambiguated(matches) + if maybe_disambiguated: + dependencies.add(maybe_disambiguated) + + logging.debug( + f"Found {pluralize(len(dependencies), 'dependency')} for target {request.field_set.address}" + ) + return InferredDependencies(dependencies) + + +def rules(): + return [ + *collect_rules(), + *renderer.rules(), + *k8s_parser.rules(), + *helm_target_types_rules(), + *docker_target_types_rules(), + UnionRule(InferDependenciesRequest, InferHelmDeploymentDependenciesRequest), + ] diff --git a/src/python/pants/backend/helm/dependency_inference/deployment_test.py b/src/python/pants/backend/helm/dependency_inference/deployment_test.py new file mode 100644 index 000000000000..3948d00c8655 --- /dev/null +++ b/src/python/pants/backend/helm/dependency_inference/deployment_test.py @@ -0,0 +1,224 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.docker.target_types import DockerImageTarget +from pants.backend.helm.dependency_inference import deployment +from pants.backend.helm.dependency_inference.deployment import ( + FirstPartyHelmDeploymentMappings, + HelmDeploymentDependenciesInferenceFieldSet, + HelmDeploymentReport, + InferHelmDeploymentDependenciesRequest, +) +from pants.backend.helm.target_types import ( + HelmChartTarget, + HelmDeploymentFieldSet, + HelmDeploymentTarget, +) +from pants.backend.helm.testutil import ( + HELM_CHART_FILE, + HELM_TEMPLATE_HELPERS_FILE, + HELM_VALUES_FILE, + K8S_SERVICE_TEMPLATE, +) +from pants.backend.helm.util_rules import chart, tool +from pants.backend.python.util_rules import pex +from pants.core.util_rules import config_files, external_tool, stripped_source_files +from pants.engine import process +from pants.engine.addresses import Address +from pants.engine.internals.graph import rules as graph_rules +from pants.engine.rules import QueryRule +from pants.engine.target import InferredDependencies +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + target_types=[HelmChartTarget, HelmDeploymentTarget, DockerImageTarget], + rules=[ + *config_files.rules(), + *external_tool.rules(), + *chart.rules(), + *deployment.rules(), + *graph_rules(), + *pex.rules(), + *process.rules(), + *stripped_source_files.rules(), + *tool.rules(), + QueryRule(FirstPartyHelmDeploymentMappings, ()), + QueryRule(HelmDeploymentReport, (HelmDeploymentFieldSet,)), + QueryRule(InferredDependencies, (InferHelmDeploymentDependenciesRequest,)), + ], + ) + + +def test_deployment_dependencies_report(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/mychart/BUILD": "helm_chart()", + "src/mychart/Chart.yaml": HELM_CHART_FILE, + "src/mychart/values.yaml": HELM_VALUES_FILE, + "src/mychart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, + "src/mychart/templates/service.yaml": K8S_SERVICE_TEMPLATE, + "src/mychart/templates/pod.yaml": dedent( + """\ + apiVersion: v1 + kind: Pod + metadata: + name: {{ template "fullname" . }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + spec: + containers: + - name: myapp-container + image: busybox:1.28 + initContainers: + - name: init-service + image: busybox:1.29 + - name: init-db + image: example.com/containers/busybox:1.28 + """ + ), + "src/deployment/BUILD": "helm_deployment(name='foo', dependencies=['//src/mychart'])", + } + ) + + source_root_patterns = ("/src/*",) + rule_runner.set_options( + [f"--source-root-patterns={repr(source_root_patterns)}"], env_inherit=PYTHON_BOOTSTRAP_ENV + ) + + target = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(target) + + dependencies_report = rule_runner.request(HelmDeploymentReport, [field_set]) + + expected_container_refs = [ + "busybox:1.28", + "busybox:1.29", + "example.com/containers/busybox:1.28", + ] + + assert len(dependencies_report.all_image_refs) == 3 + assert set(dependencies_report.all_image_refs) == set(expected_container_refs) + + +def test_inject_deployment_dependencies(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/mychart/BUILD": "helm_chart()", + "src/mychart/Chart.yaml": HELM_CHART_FILE, + "src/mychart/values.yaml": HELM_VALUES_FILE, + "src/mychart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, + "src/mychart/templates/pod.yaml": dedent( + """\ + apiVersion: v1 + kind: Pod + metadata: + name: {{ template "fullname" . }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + spec: + containers: + - name: myapp-container + image: src/image:myapp + """ + ), + "src/deployment/BUILD": "helm_deployment(name='foo', dependencies=['//src/mychart'])", + "src/image/BUILD": "docker_image(name='myapp')", + "src/image/Dockerfile": "FROM busybox:1.28", + } + ) + + source_root_patterns = ("src/*",) + rule_runner.set_options( + [f"--source-root-patterns={repr(source_root_patterns)}"], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + deployment_addr = Address("src/deployment", target_name="foo") + tgt = rule_runner.get_target(deployment_addr) + + expected_image_ref = "src/image:myapp" + expected_dependency_addr = Address("src/image", target_name="myapp") + + mappings = rule_runner.request(FirstPartyHelmDeploymentMappings, []) + assert mappings.docker_addresses_referenced_by(deployment_addr) == [ + (expected_image_ref, expected_dependency_addr) + ] + + inferred_dependencies = rule_runner.request( + InferredDependencies, + [ + InferHelmDeploymentDependenciesRequest( + HelmDeploymentDependenciesInferenceFieldSet.create(tgt) + ) + ], + ) + + assert len(inferred_dependencies.include) == 1 + assert list(inferred_dependencies.include)[0] == expected_dependency_addr + + +def test_disambiguate_docker_dependency(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/mychart/BUILD": "helm_chart()", + "src/mychart/Chart.yaml": HELM_CHART_FILE, + "src/mychart/values.yaml": HELM_VALUES_FILE, + "src/mychart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, + "src/mychart/templates/pod.yaml": dedent( + """\ + apiVersion: v1 + kind: Pod + metadata: + name: {{ template "fullname" . }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + spec: + containers: + - name: myapp-container + image: registry/image:latest + """ + ), + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name="foo", + dependencies=[ + "//src/mychart", + "!//registry/image:latest", + ] + ) + """ + ), + "registry/image/BUILD": "docker_image(name='latest')", + "registry/image/Dockerfile": "FROM busybox:1.28", + } + ) + + source_root_patterns = ("/", "src/*") + rule_runner.set_options( + [f"--source-root-patterns={repr(source_root_patterns)}"], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + deployment_addr = Address("src/deployment", target_name="foo") + tgt = rule_runner.get_target(deployment_addr) + + inferred_dependencies = rule_runner.request( + InferredDependencies, + [ + InferHelmDeploymentDependenciesRequest( + HelmDeploymentDependenciesInferenceFieldSet.create(tgt) + ) + ], + ) + + assert len(inferred_dependencies.include) == 0 diff --git a/src/python/pants/backend/helm/dependency_inference/unittest_test.py b/src/python/pants/backend/helm/dependency_inference/unittest_test.py index 89c9a1cc5183..3a64f0d126f2 100644 --- a/src/python/pants/backend/helm/dependency_inference/unittest_test.py +++ b/src/python/pants/backend/helm/dependency_inference/unittest_test.py @@ -19,7 +19,7 @@ from pants.backend.helm.testutil import ( HELM_CHART_FILE, HELM_VALUES_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, gen_chart_file, ) from pants.build_graph.address import Address @@ -50,7 +50,7 @@ def test_infers_single_chart(rule_runner: RuleRunner) -> None: ), "Chart.yaml": HELM_CHART_FILE, "values.yaml": HELM_VALUES_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, "tests/BUILD": textwrap.dedent( """\ helm_unittest_tests(name="foo_tests", sources=["*_test.yaml"]) @@ -80,13 +80,13 @@ def test_injects_parent_chart(rule_runner: RuleRunner) -> None: "src/chart1/BUILD": """helm_chart()""", "src/chart1/Chart.yaml": gen_chart_file("chart1", version="0.1.0"), "src/chart1/values.yaml": HELM_VALUES_FILE, - "src/chart1/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart1/templates/service.yaml": K8S_SERVICE_TEMPLATE, "src/chart1/tests/BUILD": """helm_unittest_tests(sources=["*_test.yaml"])""", "src/chart1/tests/service_test.yaml": "", "src/chart2/BUILD": """helm_chart()""", "src/chart2/Chart.yaml": gen_chart_file("chart2", version="0.1.0"), "src/chart2/values.yaml": HELM_VALUES_FILE, - "src/chart2/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart2/templates/service.yaml": K8S_SERVICE_TEMPLATE, "src/chart2/tests/BUILD": """helm_unittest_tests(sources=["*_test.yaml"])""", "src/chart2/tests/service_test.yaml": "", } diff --git a/src/python/pants/backend/helm/goals/deploy.py b/src/python/pants/backend/helm/goals/deploy.py new file mode 100644 index 000000000000..35635ea9d830 --- /dev/null +++ b/src/python/pants/backend/helm/goals/deploy.py @@ -0,0 +1,89 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from pants.backend.docker.goals.package_image import DockerFieldSet +from pants.backend.helm.dependency_inference import deployment +from pants.backend.helm.subsystems.helm import HelmSubsystem +from pants.backend.helm.subsystems.post_renderer import HelmPostRenderer +from pants.backend.helm.target_types import ( + HelmDeploymentFieldSet, + HelmDeploymentTarget, + HelmDeploymentTimeoutField, +) +from pants.backend.helm.util_rules import post_renderer +from pants.backend.helm.util_rules.post_renderer import HelmDeploymentPostRendererRequest +from pants.backend.helm.util_rules.renderer import HelmDeploymentCmd, HelmDeploymentRequest +from pants.core.goals.deploy import DeployFieldSet, DeployProcess +from pants.engine.process import InteractiveProcess +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import DependenciesRequest, Targets +from pants.engine.unions import UnionRule +from pants.util.docutil import bin_name +from pants.util.logging import LogLevel +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class DeployHelmDeploymentFieldSet(HelmDeploymentFieldSet, DeployFieldSet): + + timeout: HelmDeploymentTimeoutField + + +@rule(desc="Run Helm deploy process", level=LogLevel.DEBUG) +async def run_helm_deploy( + field_set: DeployHelmDeploymentFieldSet, helm_subsystem: HelmSubsystem +) -> DeployProcess: + passthrough_args = helm_subsystem.valid_args( + extra_help=softwrap( + f""" + Most invalid arguments have equivalent fields in the `{HelmDeploymentTarget.alias}` target. + Usage of fields is encouraged over passthrough arguments as that enables repeatable deployments. + + Please run `{bin_name()} help {HelmDeploymentTarget.alias}` for more information. + """ + ) + ) + + target_dependencies, post_renderer = await MultiGet( + Get(Targets, DependenciesRequest(field_set.dependencies)), + Get(HelmPostRenderer, HelmDeploymentPostRendererRequest(field_set)), + ) + + publish_targets = [tgt for tgt in target_dependencies if DockerFieldSet.is_applicable(tgt)] + + interactive_process = await Get( + InteractiveProcess, + HelmDeploymentRequest( + cmd=HelmDeploymentCmd.UPGRADE, + field_set=field_set, + extra_argv=[ + "--install", + *(("--timeout", f"{field_set.timeout.value}s") if field_set.timeout.value else ()), + *passthrough_args, + ], + post_renderer=post_renderer, + description=f"Deploying {field_set.address}", + ), + ) + + return DeployProcess( + name=field_set.address.spec, + publish_dependencies=tuple(publish_targets), + process=interactive_process, + ) + + +def rules(): + return [ + *collect_rules(), + *deployment.rules(), + *post_renderer.rules(), + UnionRule(DeployFieldSet, DeployHelmDeploymentFieldSet), + ] diff --git a/src/python/pants/backend/helm/goals/deploy_test.py b/src/python/pants/backend/helm/goals/deploy_test.py new file mode 100644 index 000000000000..9b2cf018ddba --- /dev/null +++ b/src/python/pants/backend/helm/goals/deploy_test.py @@ -0,0 +1,162 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.docker.target_types import DockerImageTarget +from pants.backend.helm.goals.deploy import DeployHelmDeploymentFieldSet +from pants.backend.helm.goals.deploy import rules as helm_deploy_rules +from pants.backend.helm.target_types import HelmChartTarget, HelmDeploymentTarget +from pants.backend.helm.testutil import HELM_CHART_FILE +from pants.backend.helm.util_rules.tool import HelmBinary +from pants.core.goals.deploy import DeployProcess +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + target_types=[HelmChartTarget, HelmDeploymentTarget, DockerImageTarget], + rules=[ + *helm_deploy_rules(), + QueryRule(HelmBinary, ()), + QueryRule(DeployProcess, (DeployHelmDeploymentFieldSet,)), + ], + ) + return rule_runner + + +def test_run_helm_deploy(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", + "src/chart/Chart.yaml": HELM_CHART_FILE, + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name="foo", + description="Foo deployment", + namespace="uat", + create_namespace=True, + skip_crds=True, + no_hooks=True, + dependencies=["//src/chart", "//src/docker/myimage"], + sources=["common.yaml", "*.yaml", "*-override.yaml", "subdir/*.yaml", "subdir/*-override.yaml", "subdir/last.yaml"], + values={ + "key": "foo", + "amount": "300", + "long_string": "This is a long string", + }, + timeout=150, + ) + """ + ), + "src/deployment/common.yaml": "", + "src/deployment/bar-override.yaml": "", + "src/deployment/foo.yaml": "", + "src/deployment/bar.yaml": "", + "src/deployment/subdir/foo.yaml": "", + "src/deployment/subdir/foo-override.yaml": "", + "src/deployment/subdir/bar.yaml": "", + "src/deployment/subdir/last.yaml": "", + "src/docker/myimage/BUILD": dedent( + """\ + docker_image(registries=["https://wwww.example.com"], repository="myimage") + """ + ), + "src/docker/myimage/Dockerfile": dedent( + """\ + FROM busybox + """ + ), + } + ) + + source_root_patterns = ["/src/*"] + deploy_args = ["--kubeconfig", "./kubeconfig"] + rule_runner.set_options( + [ + f"--source-root-patterns={repr(source_root_patterns)}", + f"--helm-args={repr(deploy_args)}", + ], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + target = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = DeployHelmDeploymentFieldSet.create(target) + + helm = rule_runner.request(HelmBinary, []) + deploy_process = rule_runner.request(DeployProcess, [field_set]) + + assert deploy_process.process + assert deploy_process.process.process.argv == ( + helm.path, + "upgrade", + "foo", + "mychart", + "--description", + '"Foo deployment"', + "--namespace", + "uat", + "--create-namespace", + "--skip-crds", + "--no-hooks", + "--post-renderer", + "./post_renderer_wrapper.sh", + "--values", + "common.yaml,bar.yaml,foo.yaml,bar-override.yaml,subdir/bar.yaml,subdir/foo.yaml,subdir/foo-override.yaml,subdir/last.yaml", + "--set", + "key=foo", + "--set", + "amount=300", + "--set", + 'long_string="This is a long string"', + "--install", + "--timeout", + "150s", + "--kubeconfig", + "./kubeconfig", + ) + + +def test_raises_error_when_using_invalid_passthrough_args(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", + "src/chart/Chart.yaml": HELM_CHART_FILE, + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name="bar", + namespace="uat", + dependencies=["//src/chart"], + sources=["*.yaml", "subdir/*.yml"] + ) + """ + ), + } + ) + + source_root_patterns = ["/src/*"] + deploy_args = ["--force", "--debug", "--kubeconfig=./kubeconfig", "--namespace", "foo"] + rule_runner.set_options( + [ + f"--source-root-patterns={repr(source_root_patterns)}", + f"--helm-args={repr(deploy_args)}", + ], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + target = rule_runner.get_target(Address("src/deployment", target_name="bar")) + field_set = DeployHelmDeploymentFieldSet.create(target) + + with pytest.raises( + ExecutionError, match="The following command line arguments are not valid: --namespace foo." + ): + rule_runner.request(DeployProcess, [field_set]) diff --git a/src/python/pants/backend/helm/goals/lint.py b/src/python/pants/backend/helm/goals/lint.py index 2956faf58d4e..15bd9a92608d 100644 --- a/src/python/pants/backend/helm/goals/lint.py +++ b/src/python/pants/backend/helm/goals/lint.py @@ -12,6 +12,7 @@ HelmChartLintStrictField, HelmSkipLintField, ) +from pants.backend.helm.util_rules import tool from pants.backend.helm.util_rules.chart import HelmChart, HelmChartRequest from pants.backend.helm.util_rules.tool import HelmProcess from pants.core.goals.lint import LintResult, LintResults, LintTargetsRequest @@ -54,7 +55,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess return HelmProcess( argv, input_digest=chart.snapshot.digest, - description=f"Linting chart: {chart.metadata.name}", + description=f"Linting chart: {chart.info.name}", ) process_results = await MultiGet( @@ -67,7 +68,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess ) results = [ LintResult.from_fallible_process_result( - process_result, partition_description=chart.metadata.name + process_result, partition_description=chart.info.name ) for chart, process_result in zip(charts, process_results) ] @@ -75,4 +76,4 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess def rules(): - return [*collect_rules(), UnionRule(LintTargetsRequest, HelmLintRequest)] + return [*collect_rules(), *tool.rules(), UnionRule(LintTargetsRequest, HelmLintRequest)] diff --git a/src/python/pants/backend/helm/goals/lint_test.py b/src/python/pants/backend/helm/goals/lint_test.py index 63870f576074..84e83946a553 100644 --- a/src/python/pants/backend/helm/goals/lint_test.py +++ b/src/python/pants/backend/helm/goals/lint_test.py @@ -15,14 +15,14 @@ from pants.backend.helm.testutil import ( HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, - K8S_INGRESS_FILE_WITH_LINT_WARNINGS, - K8S_SERVICE_FILE, + K8S_INGRESS_TEMPLATE_WITH_LINT_WARNINGS, + K8S_SERVICE_TEMPLATE, gen_chart_file, ) -from pants.backend.helm.util_rules import chart, sources, tool +from pants.backend.helm.util_rules import chart, sources from pants.build_graph.address import Address from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules import config_files, external_tool, stripped_source_files +from pants.core.util_rules import config_files, stripped_source_files from pants.engine.rules import QueryRule, SubsystemRule from pants.engine.target import Target from pants.source.source_root import rules as source_root_rules @@ -36,9 +36,7 @@ def rule_runner() -> RuleRunner: rules=[ *config_files.rules(), *chart.rules(), - *external_tool.rules(), *helm_lint_rules(), - *tool.rules(), *stripped_source_files.rules(), *source_root_rules(), *sources.rules(), @@ -74,7 +72,7 @@ def test_lint_non_strict_chart_passing(rule_runner: RuleRunner) -> None: "Chart.yaml": gen_chart_file("mychart", version="0.1.0", icon=None), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) @@ -93,7 +91,7 @@ def test_lint_non_strict_chart_failing(rule_runner: RuleRunner) -> None: "Chart.yaml": gen_chart_file("mychart", version="0.1.0", icon="wrong URL"), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) @@ -111,7 +109,7 @@ def test_lint_strict_chart_failing(rule_runner: RuleRunner) -> None: "Chart.yaml": gen_chart_file("mychart", version="0.1.0", icon=None), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/ingress.yaml": K8S_INGRESS_FILE_WITH_LINT_WARNINGS, + "templates/ingress.yaml": K8S_INGRESS_TEMPLATE_WITH_LINT_WARNINGS, } ) @@ -129,7 +127,7 @@ def test_global_lint_strict_chart_failing(rule_runner: RuleRunner) -> None: "Chart.yaml": gen_chart_file("mychart", version="0.1.0", icon=None), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/ingress.yaml": K8S_INGRESS_FILE_WITH_LINT_WARNINGS, + "templates/ingress.yaml": K8S_INGRESS_TEMPLATE_WITH_LINT_WARNINGS, } ) @@ -148,7 +146,7 @@ def test_lint_strict_chart_passing(rule_runner: RuleRunner) -> None: "Chart.yaml": gen_chart_file("mychart", version="0.1.0", icon=None), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) @@ -166,12 +164,12 @@ def test_one_lint_result_per_chart(rule_runner: RuleRunner) -> None: "src/chart1/Chart.yaml": gen_chart_file("chart1", version="0.1.0"), "src/chart1/values.yaml": HELM_VALUES_FILE, "src/chart1/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "src/chart1/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart1/templates/service.yaml": K8S_SERVICE_TEMPLATE, "src/chart2/BUILD": "helm_chart()", "src/chart2/Chart.yaml": gen_chart_file("chart2", version="0.1.0"), "src/chart2/values.yaml": HELM_VALUES_FILE, "src/chart2/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "src/chart2/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart2/templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) source_root_patterns = ("src/*",) @@ -196,7 +194,7 @@ def test_skip_lint(rule_runner: RuleRunner) -> None: "src/chart/Chart.yaml": gen_chart_file("chart", version="0.1.0"), "src/chart/values.yaml": HELM_VALUES_FILE, "src/chart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "src/chart/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart/templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) diff --git a/src/python/pants/backend/helm/goals/package.py b/src/python/pants/backend/helm/goals/package.py index af534a0a8aaf..85faf708556b 100644 --- a/src/python/pants/backend/helm/goals/package.py +++ b/src/python/pants/backend/helm/goals/package.py @@ -31,13 +31,13 @@ @dataclass(frozen=True) class BuiltHelmArtifact(BuiltPackageArtifact): - metadata: HelmChartMetadata | None = None + info: HelmChartMetadata | None = None @classmethod - def create(cls, relpath: str, metadata: HelmChartMetadata) -> BuiltHelmArtifact: + def create(cls, relpath: str, info: HelmChartMetadata) -> BuiltHelmArtifact: return cls( relpath=relpath, - metadata=metadata, + info=info, extra_log_lines=(f"Built Helm chart artifact: {relpath}",), ) @@ -57,7 +57,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage: ) input_digest = await Get(Digest, MergeDigests([chart.snapshot.digest, result_digest])) - process_output_file = os.path.join(result_dir, f"{chart.metadata.artifact_name}.tgz") + process_output_file = os.path.join(result_dir, f"{chart.info.artifact_name}.tgz") process_result = await Get( ProcessResult, @@ -80,7 +80,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage: return BuiltPackage( final_snapshot.digest, artifacts=tuple( - BuiltHelmArtifact.create(file, chart.metadata) for file in final_snapshot.files + BuiltHelmArtifact.create(file, chart.info) for file in final_snapshot.files ), ) diff --git a/src/python/pants/backend/helm/goals/package_test.py b/src/python/pants/backend/helm/goals/package_test.py index 8aab3a06f3c5..068888751dca 100644 --- a/src/python/pants/backend/helm/goals/package_test.py +++ b/src/python/pants/backend/helm/goals/package_test.py @@ -15,7 +15,7 @@ from pants.backend.helm.testutil import ( HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, gen_chart_file, ) from pants.backend.helm.util_rules import chart, sources, tool @@ -61,7 +61,7 @@ def _assert_build_package(rule_runner: RuleRunner, *, chart_name: str, chart_ver assert result.artifacts[0].relpath == os.path.join( dest_dir, f"{chart_name}-{chart_version}.tgz" ) - assert result.artifacts[0].metadata + assert result.artifacts[0].info def test_helm_package(rule_runner: RuleRunner) -> None: @@ -74,7 +74,7 @@ def test_helm_package(rule_runner: RuleRunner) -> None: f"src/{chart_name}/Chart.yaml": gen_chart_file(chart_name, version=chart_version), f"src/{chart_name}/values.yaml": HELM_VALUES_FILE, f"src/{chart_name}/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - f"src/{chart_name}/templates/service.yaml": K8S_SERVICE_FILE, + f"src/{chart_name}/templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) @@ -93,7 +93,7 @@ def test_helm_package_with_custom_output_path(rule_runner: RuleRunner) -> None: f"src/{chart_name}/Chart.yaml": gen_chart_file(chart_name, version=chart_version), f"src/{chart_name}/values.yaml": HELM_VALUES_FILE, f"src/{chart_name}/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - f"src/{chart_name}/templates/service.yaml": K8S_SERVICE_FILE, + f"src/{chart_name}/templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) diff --git a/src/python/pants/backend/helm/goals/publish.py b/src/python/pants/backend/helm/goals/publish.py index b4307ebbda2e..54b5472e5918 100644 --- a/src/python/pants/backend/helm/goals/publish.py +++ b/src/python/pants/backend/helm/goals/publish.py @@ -22,7 +22,7 @@ PublishProcesses, PublishRequest, ) -from pants.engine.process import InteractiveProcess, InteractiveProcessRequest, Process +from pants.engine.process import InteractiveProcess, Process from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.logging import LogLevel @@ -57,10 +57,10 @@ async def publish_helm_chart( ) -> PublishProcesses: remotes = helm_subsystem.remotes() built_artifacts = [ - (pkg, artifact, artifact.metadata) + (pkg, artifact, artifact.info) for pkg in request.packages for artifact in pkg.artifacts - if isinstance(artifact, BuiltHelmArtifact) and artifact.metadata + if isinstance(artifact, BuiltHelmArtifact) and artifact.info ] registries_to_push = list(remotes.get(*(request.field_set.registries.value or []))) @@ -106,15 +106,10 @@ async def publish_helm_chart( for registry in registries_to_push ) - interactive_processes = await MultiGet( - Get(InteractiveProcess, InteractiveProcessRequest(process)) for process in processes - ) - - refs_and_processes = zip(publish_refs, interactive_processes) return PublishProcesses( [ - PublishPackages(names=(package_ref,), process=process) - for package_ref, process in refs_and_processes + PublishPackages(names=(package_ref,), process=InteractiveProcess.from_process(process)) + for package_ref, process in zip(publish_refs, processes) ] ) diff --git a/src/python/pants/backend/helm/resolve/fetch.py b/src/python/pants/backend/helm/resolve/fetch.py index 034bd4a6491b..258dd4e0b6e4 100644 --- a/src/python/pants/backend/helm/resolve/fetch.py +++ b/src/python/pants/backend/helm/resolve/fetch.py @@ -11,6 +11,7 @@ from pants.backend.helm.resolve import artifacts from pants.backend.helm.resolve.artifacts import HelmArtifact, ResolvedHelmArtifact from pants.backend.helm.target_types import HelmArtifactFieldSet +from pants.backend.helm.util_rules import tool from pants.backend.helm.util_rules.tool import HelmProcess from pants.engine.addresses import Address from pants.engine.collection import Collection @@ -137,4 +138,4 @@ def create_fetch_process(artifact: ResolvedHelmArtifact) -> HelmProcess: def rules(): - return [*collect_rules(), *artifacts.rules()] + return [*collect_rules(), *artifacts.rules(), *tool.rules()] diff --git a/src/python/pants/backend/helm/subsystems/BUILD b/src/python/pants/backend/helm/subsystems/BUILD index f9b4a7be9f0f..7673183676f2 100644 --- a/src/python/pants/backend/helm/subsystems/BUILD +++ b/src/python/pants/backend/helm/subsystems/BUILD @@ -1,6 +1,42 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + sources=["*.py", "!*_test.py", "!post_renderer_main.py", "!k8s_parser_main.py"], + dependencies=[":post_renderer", ":k8s_parser"], +) -python_tests(name="tests") \ No newline at end of file +python_tests(name="tests") + +# Post-Renderer + +python_requirement( + name="yamlpath", + requirements=[ + "yamlpath>=3.6,<3.7", + "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21", + ], + resolve="helm-post-renderer", +) + +python_sources( + name="__post_renderer_main", + sources=["post_renderer_main.py"], + resolve="helm-post-renderer", + skip_mypy=True, +) + +resources(name="post_renderer", sources=["post_renderer.lock", "post_renderer_main.py"]) + +# Kubernetes manifest parser + +python_requirement(name="k8s", requirements=["hikaru==0.11.0b"], resolve="helm-k8s-parser") + +python_sources( + name="__k8s_parser_main", + sources=["k8s_parser_main.py"], + resolve="helm-k8s-parser", + skip_mypy=True, +) + +resources(name="k8s_parser", sources=["k8s_parser.lock", "k8s_parser_main.py"]) diff --git a/src/python/pants/backend/helm/subsystems/helm.py b/src/python/pants/backend/helm/subsystems/helm.py index e37a65f9ef09..683181b91fcc 100644 --- a/src/python/pants/backend/helm/subsystems/helm.py +++ b/src/python/pants/backend/helm/subsystems/helm.py @@ -4,15 +4,53 @@ from __future__ import annotations import os -from typing import Any +from typing import Any, Iterable from pants.backend.helm.resolve.remotes import HelmRemotes from pants.backend.helm.target_types import HelmChartTarget, HelmRegistriesField from pants.core.util_rules.external_tool import TemplatedExternalTool from pants.engine.platform import Platform -from pants.option.option_types import BoolOption, DictOption, StrOption +from pants.option.option_types import ArgsListOption, BoolOption, DictOption, StrOption from pants.util.memo import memoized_method -from pants.util.strutil import softwrap +from pants.util.strutil import bullet_list, softwrap + +_VALID_PASSTHROUGH_FLAGS = [ + "--atomic", + "--dry-run", + "--debug", + "--force", + "--replace", + "--wait", + "--wait-for-jobs", +] + +_VALID_PASSTHROUGH_OPTS = [ + "--kubeconfig", + "--kube-context", + "--kube-apiserver", + "--kube-as-group", + "--kube-as-user", + "--kube-ca-file", + "--kube-token", +] + + +class InvalidHelmPassthroughArgs(Exception): + def __init__(self, args: Iterable[str], *, extra_help: str = "") -> None: + super().__init__( + softwrap( + f""" + The following command line arguments are not valid: {' '.join(args)}. + + Only the following passthrough arguments are allowed: + + {bullet_list([*_VALID_PASSTHROUGH_FLAGS, *_VALID_PASSTHROUGH_OPTS])} + + {extra_help} + """ + ) + ) + registries_help = softwrap( f""" @@ -79,6 +117,33 @@ class HelmSubsystem(TemplatedExternalTool): advanced=True, ) + args = ArgsListOption( + example="--dry-run", + passthrough=True, + extra_help=softwrap( + f""" + Additional arguments to pass to Helm command line. + + Only a subset of Helm arguments are considered valid as passthrough arguments as most of them + have equivalents in the form of fields of the different target types. + + The list of valid arguments is as folows: + + {bullet_list([*_VALID_PASSTHROUGH_FLAGS, *_VALID_PASSTHROUGH_OPTS])} + + Before attempting to use passthrough arguments, check the refence of each of the available target types + to see what fields are accepted in each of them. + """ + ), + ) + + @memoized_method + def valid_args(self, *, extra_help: str = "") -> tuple[str, ...]: + valid, invalid = _cleanup_passthrough_args(self.args) + if invalid: + raise InvalidHelmPassthroughArgs(invalid, extra_help=extra_help) + return tuple(valid) + def generate_exe(self, plat: Platform) -> str: mapped_plat = self.default_url_platform_mapping[plat.value] bin_path = os.path.join(mapped_plat, "helm") @@ -87,3 +152,27 @@ def generate_exe(self, plat: Platform) -> str: @memoized_method def remotes(self) -> HelmRemotes: return HelmRemotes.from_dict(self._registries) + + +def _cleanup_passthrough_args(args: Iterable[str]) -> tuple[list[str], list[str]]: + valid_args: list[str] = [] + removed_args: list[str] = [] + + skip = False + for arg in args: + if skip: + valid_args.append(arg) + skip = False + continue + + if arg in _VALID_PASSTHROUGH_FLAGS: + valid_args.append(arg) + elif "=" in arg and arg.split("=")[0] in _VALID_PASSTHROUGH_OPTS: + valid_args.append(arg) + elif arg in _VALID_PASSTHROUGH_OPTS: + valid_args.append(arg) + skip = True + else: + removed_args.append(arg) + + return (valid_args, removed_args) diff --git a/src/python/pants/backend/helm/subsystems/k8s_parser.lock b/src/python/pants/backend/helm/subsystems/k8s_parser.lock new file mode 100644 index 000000000000..e7534601719f --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/k8s_parser.lock @@ -0,0 +1,1023 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// build-support/bin/generate_all_lockfiles.sh +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 2, +// "valid_for_interpreter_constraints": [ +// "CPython<3.10,>=3.7" +// ], +// "generated_with_requirements": [ +// "hikaru==0.11.0b" +// ] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f", + "url": "https://files.pythonhosted.org/packages/39/3a/cd60ecce0d9737efefc06a074ae280a5d0e904d697fe59b414bf8ab5c472/autopep8-1.6.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "url": "https://files.pythonhosted.org/packages/ec/67/564f7d15712a84d4035aa5ad0b97eeafdeccdb7e806d6a822595bf0ffa5f/autopep8-1.6.0.tar.gz" + } + ], + "project_name": "autopep8", + "requires_dists": [ + "pycodestyle>=2.8.0", + "toml" + ], + "requires_python": null, + "version": "1.6" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3524739d76b6b3ed1132422bf9d82123cd1705086723bc3e235ca39fd21c667d", + "url": "https://files.pythonhosted.org/packages/a5/59/bd6d44da2b364fd2bd7a0b2ce2edfe200b79faad1cde14ce5ef13d504393/black-22.1.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "fdb8754b453fb15fad3f72cd9cad3e16776f0964d67cf30ebcbf10327a3777a3", + "url": "https://files.pythonhosted.org/packages/08/b2/dbd7330ffe13571e17b7f905f7639ba77f01282ff1ecd94f3278c50ebb32/black-22.1.0-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "f5660feab44c2e3cb24b2419b998846cbb01c23c7fe645fee45087efa3da2d61", + "url": "https://files.pythonhosted.org/packages/0b/7f/384cf21254346f4cd535fa8bf2531ff2b3f1307680199e28ea949c3ecb89/black-22.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "ccad888050f5393f0d6029deea2a33e5ae371fd182a697313bdbd835d3edaf9c", + "url": "https://files.pythonhosted.org/packages/38/95/e3f3796278da6c399003db92d3254f330f928777230cda43a3607dc0f913/black-22.1.0-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "2d6f331c02f0f40aa51a22e479c8209d37fcd520c77721c034517d44eecf5912", + "url": "https://files.pythonhosted.org/packages/3e/c4/95eea7bd67b37c54b7322ff3595fd3d679345e2b89ceca48fe3ec10df52c/black-22.1.0-cp38-cp38-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5", + "url": "https://files.pythonhosted.org/packages/42/58/8a3443a5034685152270f9012a9d196c9f165791ed3f2777307708b15f6c/black-22.1.0.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "5fa1db02410b1924b6749c245ab38d30621564e658297484952f3d8a39fce7e8", + "url": "https://files.pythonhosted.org/packages/54/d7/d1f9009f3695faa1e18b53fbf17419b51b56f4cf00e5ebb7133744f29284/black-22.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "07e5c049442d7ca1a2fc273c79d1aecbbf1bc858f62e8184abe1ad175c4f7cc2", + "url": "https://files.pythonhosted.org/packages/56/25/c625a190347b5f6d940cfdeeb15958c04436328c29dc17b5bafb6dafa3ec/black-22.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "dae63f2dbf82882fa3b2a3c49c32bffe144970a573cd68d247af6560fc493ae1", + "url": "https://files.pythonhosted.org/packages/94/37/89d9866a8a5b4a5277478c9652400a38972168fb039ac9ab31b1fd87ec75/black-22.1.0-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "742ce9af3086e5bd07e58c8feb09dbb2b047b7f566eb5f5bc63fd455814979f3", + "url": "https://files.pythonhosted.org/packages/9b/78/42a83acaf953b3ea5d6067c72f795a4df4b3eb540123cc2a59ec797d174b/black-22.1.0-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8871fcb4b447206904932b54b567923e5be802b9b19b744fdff092bd2f3118d0", + "url": "https://files.pythonhosted.org/packages/a6/5e/5e3d6145ae5c8127abe1734878fff2ca6a494799cfa18fe585c33cae9198/black-22.1.0-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "efbadd9b52c060a8fc3b9658744091cb33c31f830b3f074422ed27bad2b18e8f", + "url": "https://files.pythonhosted.org/packages/c2/e2/6198c928e9cee46233463f30a8faf39a5752e75c07c8d30a908865a05a51/black-22.1.0-cp39-cp39-macosx_10_9_universal2.whl" + } + ], + "project_name": "black", + "requires_dists": [ + "aiohttp>=3.7.4; extra == \"d\"", + "click>=8.0.0", + "colorama>=0.4.3; extra == \"colorama\"", + "dataclasses>=0.6; python_version < \"3.7\"", + "ipython>=7.8.0; extra == \"jupyter\"", + "mypy-extensions>=0.4.3", + "pathspec>=0.9.0", + "platformdirs>=2", + "tokenize-rt>=3.2.0; extra == \"jupyter\"", + "tomli>=1.1.0", + "typed-ast>=1.4.2; python_version < \"3.8\" and implementation_name == \"cpython\"", + "typing-extensions>=3.10.0.0; python_version < \"3.10\"", + "uvloop>=0.15.2; extra == \"uvloop\"" + ], + "requires_python": ">=3.6.2", + "version": "22.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db", + "url": "https://files.pythonhosted.org/packages/68/aa/5fc646cae6e997c3adf3b0a7e257cda75cff21fcba15354dffd67789b7bb/cachetools-5.2.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757", + "url": "https://files.pythonhosted.org/packages/c2/6f/278225c5a070a18a76f85db5f1238f66476579fa9b04cda3722331dcc90f/cachetools-5.2.0.tar.gz" + } + ], + "project_name": "cachetools", + "requires_dists": [], + "requires_python": "~=3.7", + "version": "5.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412", + "url": "https://files.pythonhosted.org/packages/e9/06/d3d367b7af6305b16f0d28ae2aaeb86154fa91f144f036c2d5002a5a202b/certifi-2022.6.15-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "url": "https://files.pythonhosted.org/packages/cc/85/319a8a684e8ac6d87a1193090e06b6bbb302717496380e225ee10487c888/certifi-2022.6.15.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2022.6.15" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "url": "https://files.pythonhosted.org/packages/94/69/64b11e8c2fb21f08634468caef885112e682b0ebe2908e74d3616eb1c113/charset_normalizer-2.1.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413", + "url": "https://files.pythonhosted.org/packages/93/1d/d9392056df6670ae2a29fcb04cfa5cee9f6fbde7311a1bb511d4115e9b7a/charset-normalizer-2.1.0.tar.gz" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [ + "unicodedata2; extra == \"unicode_backport\"" + ], + "requires_python": ">=3.6.0", + "version": "2.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48", + "url": "https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "url": "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz" + } + ], + "project_name": "click", + "requires_dists": [ + "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"" + ], + "requires_python": ">=3.7", + "version": "8.1.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "5a7eed0cb0e3a83989fad0b59fe1329dfc8c479543039cd6fd1e01e9adf39475", + "url": "https://files.pythonhosted.org/packages/7b/17/0b14f55fc8ff002b92e2deb796dd9e28a65ca1a6272d9d844e99051afb67/google_auth-2.9.1-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "14292fa3429f2bb1e99862554cde1ee730d6840ebae067814d3d15d8549c0888", + "url": "https://files.pythonhosted.org/packages/1a/90/0a278aeed278363ab26ccb23529092b48ee3eff8867472236a1b5141f626/google-auth-2.9.1.tar.gz" + } + ], + "project_name": "google-auth", + "requires_dists": [ + "aiohttp<4.0.0dev,>=3.6.2; python_version >= \"3.6\" and extra == \"aiohttp\"", + "cachetools<6.0,>=2.0.0", + "cryptography==36.0.2; extra == \"enterprise_cert\"", + "enum34>=1.1.10; python_version < \"3.4\"", + "pyasn1-modules>=0.2.1", + "pyopenssl==22.0.0; extra == \"enterprise_cert\"", + "pyopenssl>=20.0.0; extra == \"pyopenssl\"", + "pyu2f>=0.1.5; extra == \"reauth\"", + "requests<3.0.0dev,>=2.20.0; extra == \"aiohttp\"", + "rsa<4.6; python_version < \"3.6\"", + "rsa<5,>=3.1.4; python_version >= \"3.6\"", + "six>=1.9.0" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", + "version": "2.9.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "4c4dc83fae7f97bd3d1ac3bd8a9705321dbc12252c8a6910a1f7d94ca935b308", + "url": "https://files.pythonhosted.org/packages/c5/85/df57ceaaca57f4127c8de426cbfef2d0f4227176f18e0c046eb6cb568cb3/hikaru-0.11.0b0-py3-none-any.whl" + } + ], + "project_name": "hikaru", + "requires_dists": [ + "autopep8>=1.5.5", + "black<=22.1.0,>=20.8b1", + "kubernetes<=21.7.0,>=18.20.0", + "ruamel.yaml>=0.16.12" + ], + "requires_python": null, + "version": "0.11b0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "url": "https://files.pythonhosted.org/packages/04/a2/d918dcd22354d8958fe113e1a3630137e0fc8b44859ade3063982eacd2a4/idna-3.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d", + "url": "https://files.pythonhosted.org/packages/62/08/e3fc7c8161090f742f504f40b1bccbfc544d4a4e09eb774bf40aafce5436/idna-3.3.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "3.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23", + "url": "https://files.pythonhosted.org/packages/d2/a2/8c239dc898138f208dd14b441b196e7b3032b94d3137d9d8453e186967fc/importlib_metadata-4.12.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670", + "url": "https://files.pythonhosted.org/packages/1a/16/441080c907df829016729e71d8bdd42d99b9bdde48b01492ed08912c0aa9/importlib_metadata-4.12.0.tar.gz" + } + ], + "project_name": "importlib-metadata", + "requires_dists": [ + "flufl.flake8; extra == \"testing\"", + "importlib-resources>=1.3; python_version < \"3.9\" and extra == \"testing\"", + "ipython; extra == \"perf\"", + "jaraco.packaging>=9; extra == \"docs\"", + "packaging; extra == \"testing\"", + "pyfakefs; extra == \"testing\"", + "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-checkdocs>=2.4; extra == \"testing\"", + "pytest-cov; extra == \"testing\"", + "pytest-enabler>=1.3; extra == \"testing\"", + "pytest-flake8; extra == \"testing\"", + "pytest-mypy>=0.9.1; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-perf>=0.9.2; extra == \"testing\"", + "pytest>=6; extra == \"testing\"", + "rst.linker>=1.9; extra == \"docs\"", + "sphinx; extra == \"docs\"", + "typing-extensions>=3.6.4; python_version < \"3.8\"", + "zipp>=0.5" + ], + "requires_python": ">=3.7", + "version": "4.12" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "044c20253f8577491a87af8f9edea1f929ed6d62ce306376a6cb8aed24e572c5", + "url": "https://files.pythonhosted.org/packages/ec/73/aa291e48896cb2b60d8da9907df6d10cbc08c0d6685ae9a2140ac37f8628/kubernetes-21.7.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c9849afc2eafdce60efa210049ee7a94e7ef6cf3a7afa14a69b3bf0447825977", + "url": "https://files.pythonhosted.org/packages/9b/e4/de04b848035d92acdd84d99278f021975d2beb81e393fa9cbffbffca42ad/kubernetes-21.7.0.tar.gz" + } + ], + "project_name": "kubernetes", + "requires_dists": [ + "adal>=1.0.2; extra == \"adal\"", + "certifi>=14.05.14", + "google-auth>=1.0.1", + "ipaddress>=1.0.17; python_version == \"2.7\"", + "python-dateutil>=2.5.3", + "pyyaml>=5.4.1", + "requests", + "requests-oauthlib", + "setuptools>=21.0.0", + "six>=1.9.0", + "urllib3>=1.24.2", + "websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0" + ], + "requires_python": ">=3.6", + "version": "21.7" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "url": "https://files.pythonhosted.org/packages/5c/eb/975c7c080f3223a5cdaff09612f3a5221e4ba534f7039db34c35d95fa6a5/mypy_extensions-0.4.3-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8", + "url": "https://files.pythonhosted.org/packages/63/60/0582ce2eaced55f65a4406fc97beba256de4b7a95a0034c6576458c6519f/mypy_extensions-0.4.3.tar.gz" + } + ], + "project_name": "mypy-extensions", + "requires_dists": [ + "typing>=3.5.3; python_version < \"3.5\"" + ], + "requires_python": null, + "version": "0.4.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe", + "url": "https://files.pythonhosted.org/packages/1d/46/5ee2475e1b46a26ca0fa10d3c1d479577fde6ee289f8c6aa6d7ec33e31fd/oauthlib-3.2.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2", + "url": "https://files.pythonhosted.org/packages/6e/7e/a43cec8b2df28b6494a865324f0ac4be213cb2edcf1e2a717547a93279b0/oauthlib-3.2.0.tar.gz" + } + ], + "project_name": "oauthlib", + "requires_dists": [ + "blinker>=1.4.0; extra == \"signals\"", + "cryptography>=3.0.0; extra == \"rsa\"", + "cryptography>=3.0.0; extra == \"signedtoken\"", + "pyjwt<3,>=2.0.0; extra == \"signedtoken\"" + ], + "requires_python": ">=3.6", + "version": "3.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a", + "url": "https://files.pythonhosted.org/packages/42/ba/a9d64c7bcbc7e3e8e5f93a52721b377e994c22d16196e2b0f1236774353a/pathspec-0.9.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1", + "url": "https://files.pythonhosted.org/packages/f6/33/436c5cb94e9f8902e59d1d544eb298b83c84b9ec37b5b769c5a0ad6edb19/pathspec-0.9.0.tar.gz" + } + ], + "project_name": "pathspec", + "requires_dists": [], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7", + "version": "0.9" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "url": "https://files.pythonhosted.org/packages/ed/22/967181c94c3a4063fe64e15331b4cb366bdd7dfbf46fcb8ad89650026fec/platformdirs-2.5.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19", + "url": "https://files.pythonhosted.org/packages/ff/7b/3613df51e6afbf2306fc2465671c03390229b55e3ef3ab9dd3f846a53be6/platformdirs-2.5.2.tar.gz" + } + ], + "project_name": "platformdirs", + "requires_dists": [ + "appdirs==1.4.4; extra == \"test\"", + "furo>=2021.7.5b38; extra == \"docs\"", + "proselint>=0.10.2; extra == \"docs\"", + "pytest-cov>=2.7; extra == \"test\"", + "pytest-mock>=3.6; extra == \"test\"", + "pytest>=6; extra == \"test\"", + "sphinx-autodoc-typehints>=1.12; extra == \"docs\"", + "sphinx>=4; extra == \"docs\"" + ], + "requires_python": ">=3.7", + "version": "2.5.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "url": "https://files.pythonhosted.org/packages/62/1e/a94a8d635fa3ce4cfc7f506003548d0a2447ae76fd5ca53932970fe3053f/pyasn1-0.4.8-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", + "url": "https://files.pythonhosted.org/packages/a4/db/fffec68299e6d7bad3d504147f9094830b704527a7fc098b721d38cc7fa7/pyasn1-0.4.8.tar.gz" + } + ], + "project_name": "pyasn1", + "requires_dists": [], + "requires_python": null, + "version": "0.4.8" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", + "url": "https://files.pythonhosted.org/packages/95/de/214830a981892a3e286c3794f41ae67a4495df1108c3da8a9f62159b9a9d/pyasn1_modules-0.2.8-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", + "url": "https://files.pythonhosted.org/packages/88/87/72eb9ccf8a58021c542de2588a867dbefc7556e14b2866d1e40e9e2b587e/pyasn1-modules-0.2.8.tar.gz" + } + ], + "project_name": "pyasn1-modules", + "requires_dists": [ + "pyasn1<0.5.0,>=0.4.6" + ], + "requires_python": null, + "version": "0.2.8" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b", + "url": "https://files.pythonhosted.org/packages/67/e4/fc77f1039c34b3612c4867b69cbb2b8a4e569720b1f19b0637002ee03aff/pycodestyle-2.9.1-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785", + "url": "https://files.pythonhosted.org/packages/b6/83/5bcaedba1f47200f0665ceb07bcb00e2be123192742ee0edfb66b600e5fd/pycodestyle-2.9.1.tar.gz" + } + ], + "project_name": "pycodestyle", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2.9.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9", + "url": "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "url": "https://files.pythonhosted.org/packages/4c/c4/13b4776ea2d76c115c1d1b84579f3764ee6d57204f6be27119f13a61d0a9/python-dateutil-2.8.2.tar.gz" + } + ], + "project_name": "python-dateutil", + "requires_dists": [ + "six>=1.5" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "2.8.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "url": "https://files.pythonhosted.org/packages/12/fc/a4d5a7554e0067677823f7265cb3ae22aed8a238560b5133b58cda252dad/PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "url": "https://files.pythonhosted.org/packages/21/67/b42191239c5650c9e419c4a08a7a022bbf1abf55b0391c380a72c3af5462/PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "url": "https://files.pythonhosted.org/packages/36/2b/61d51a2c4f25ef062ae3f74576b01638bebad5e045f747ff12643df63844/PyYAML-6.0.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "url": "https://files.pythonhosted.org/packages/63/6b/f5dc7942bac17192f4ef00b2d0cdd1ae45eea453d05c1944c0573debe945/PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "url": "https://files.pythonhosted.org/packages/67/d4/b95266228a25ef5bd70984c08b4efce2c035a4baa5ccafa827b266e3dc36/PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "url": "https://files.pythonhosted.org/packages/6c/3d/524c642f3db37e7e7ab8d13a3f8b0c72d04a619abc19100097d987378fc6/PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "url": "https://files.pythonhosted.org/packages/77/da/e845437ffe0dffae4e7562faf23a4f264d886431c5d2a2816c853288dc8e/PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "url": "https://files.pythonhosted.org/packages/81/59/561f7e46916b78f3c4cab8d0c307c81656f11e32c846c0c97fda0019ed76/PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "url": "https://files.pythonhosted.org/packages/9d/f6/7e91fbb58c9ee528759aea5892e062cccb426720c5830ddcce92eba00ff1/PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "url": "https://files.pythonhosted.org/packages/d7/42/7ad4b6d67a16229496d4f6e74201bdbebcf4bc1e87d5a70c9297d4961bd2/PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "url": "https://files.pythonhosted.org/packages/db/4e/74bc723f2d22677387ab90cd9139e62874d14211be7172ed8c9f9a7c81a9/PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "url": "https://files.pythonhosted.org/packages/df/75/ee0565bbf65133e5b6ffa154db43544af96ea4c42439e6b58c1e0eb44b4e/PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "url": "https://files.pythonhosted.org/packages/eb/5f/6e6fe6904e1a9c67bc2ca5629a69e7a5a0b17f079da838bab98a1e548b25/PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "url": "https://files.pythonhosted.org/packages/f5/6f/b8b4515346af7c33d3b07cd8ca8ea0700ca72e8d7a750b2b87ac0268ca4e/PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl" + } + ], + "project_name": "pyyaml", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "6" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349", + "url": "https://files.pythonhosted.org/packages/ca/91/6d9b8ccacd0412c08820f72cebaa4f0c0441b5cda699c90f618b6f8a1b42/requests-2.28.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "url": "https://files.pythonhosted.org/packages/a5/61/a867851fd5ab77277495a8709ddda0861b28163c4613b011bc00228cc724/requests-2.28.1.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use_chardet_on_py3\"", + "charset-normalizer<3,>=2", + "idna<4,>=2.5", + "urllib3<1.27,>=1.21.1" + ], + "requires_python": "<4,>=3.7", + "version": "2.28.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", + "url": "https://files.pythonhosted.org/packages/6f/bb/5deac77a9af870143c684ab46a7934038a53eb4aa975bc0687ed6ca2c610/requests_oauthlib-1.3.1-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a", + "url": "https://files.pythonhosted.org/packages/95/52/531ef197b426646f26b53815a7d2a67cb7a331ef098bb276db26a68ac49f/requests-oauthlib-1.3.1.tar.gz" + } + ], + "project_name": "requests-oauthlib", + "requires_dists": [ + "oauthlib>=3.0.0", + "oauthlib[signedtoken]>=3.0.0; extra == \"rsa\"", + "requests>=2.0.0" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7", + "version": "1.3.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", + "url": "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21", + "url": "https://files.pythonhosted.org/packages/aa/65/7d973b89c4d2351d7fb232c2e452547ddfa243e93131e7cfa766da627b52/rsa-4.9.tar.gz" + } + ], + "project_name": "rsa", + "requires_dists": [ + "pyasn1>=0.1.3" + ], + "requires_python": "<4,>=3.6", + "version": "4.9" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7", + "url": "https://files.pythonhosted.org/packages/9e/cb/938214ac358fbef7058343b3765c79a1b7ed0c366f7f992ce7ff38335652/ruamel.yaml-0.17.21-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af", + "url": "https://files.pythonhosted.org/packages/46/a9/6ed24832095b692a8cecc323230ce2ec3480015fbfa4b79941bd41b23a3c/ruamel.yaml-0.17.21.tar.gz" + } + ], + "project_name": "ruamel-yaml", + "requires_dists": [ + "ruamel.yaml.clib>=0.2.6; platform_python_implementation == \"CPython\" and python_version < \"3.11\"", + "ruamel.yaml.jinja2>=0.2; extra == \"jinja2\"", + "ryd; extra == \"docs\"" + ], + "requires_python": ">=3", + "version": "0.17.21" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0", + "url": "https://files.pythonhosted.org/packages/3e/36/f1e3b5a0507662a66f156518457ffaf530c818f204467a5c532fc44056f9/ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84", + "url": "https://files.pythonhosted.org/packages/15/7c/e65492dc1c311655760fb20a9f2512f419403fcdc9ada6c63f44d7fe7062/ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233", + "url": "https://files.pythonhosted.org/packages/23/2e/79d684c6cfa50b593f47938fec86f7c5d0208e0ecd278eef2ff0e10889d3/ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd", + "url": "https://files.pythonhosted.org/packages/8b/25/08e5ad2431a028d0723ca5540b3af6a32f58f25e83c6dda4d0fcef7288a3/ruamel.yaml.clib-0.2.6.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99", + "url": "https://files.pythonhosted.org/packages/98/8a/ba37489b423916162b086b01c7c18001cf297350694180468e1698085c58/ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed", + "url": "https://files.pythonhosted.org/packages/ba/2c/076d00f31f9476ccad3a6a5446ee30c5f0921012d714c76f3111e29b06ab/ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd", + "url": "https://files.pythonhosted.org/packages/d1/17/630d1d28e0fc442115280f3928b8a2b78a47b5c75bb619d16bfc4d046a69/ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl" + } + ], + "project_name": "ruamel-yaml-clib", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "0.2.6" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6", + "url": "https://files.pythonhosted.org/packages/90/2e/109766d7f3cb854a083dd66bfc0bf2bf9e40f373829efedb4b5e0d104aa3/setuptools-63.4.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca", + "url": "https://files.pythonhosted.org/packages/63/2e/e1f3e1b02d7ff0baa356413e93ad8a88706accd6b3538e18204a8a00c42d/setuptools-63.4.1.tar.gz" + } + ], + "project_name": "setuptools", + "requires_dists": [ + "build[virtualenv]; extra == \"testing\"", + "build[virtualenv]; extra == \"testing-integration\"", + "filelock>=3.4.0; extra == \"testing\"", + "filelock>=3.4.0; extra == \"testing-integration\"", + "flake8-2020; extra == \"testing\"", + "flake8<5; extra == \"testing\"", + "furo; extra == \"docs\"", + "ini2toml[lite]>=0.9; extra == \"testing\"", + "jaraco.envs>=2.2; extra == \"testing\"", + "jaraco.envs>=2.2; extra == \"testing-integration\"", + "jaraco.packaging>=9; extra == \"docs\"", + "jaraco.path>=3.2.0; extra == \"testing\"", + "jaraco.path>=3.2.0; extra == \"testing-integration\"", + "jaraco.tidelift>=1.4; extra == \"docs\"", + "mock; extra == \"testing\"", + "pip-run>=8.8; extra == \"testing\"", + "pip>=19.1; extra == \"testing\"", + "pygments-github-lexers==0.0.5; extra == \"docs\"", + "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-checkdocs>=2.4; extra == \"testing\"", + "pytest-cov; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-enabler; extra == \"testing-integration\"", + "pytest-enabler>=1.3; extra == \"testing\"", + "pytest-flake8; extra == \"testing\"", + "pytest-mypy>=0.9.1; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-perf; extra == \"testing\"", + "pytest-xdist; extra == \"testing\"", + "pytest-xdist; extra == \"testing-integration\"", + "pytest; extra == \"testing-integration\"", + "pytest>=6; extra == \"testing\"", + "rst.linker>=1.9; extra == \"docs\"", + "sphinx-favicon; extra == \"docs\"", + "sphinx-hoverxref<2; extra == \"docs\"", + "sphinx-inline-tabs; extra == \"docs\"", + "sphinx-notfound-page==0.8.3; extra == \"docs\"", + "sphinx-reredirects; extra == \"docs\"", + "sphinx; extra == \"docs\"", + "sphinxcontrib-towncrier; extra == \"docs\"", + "tomli-w>=1.0.0; extra == \"testing\"", + "tomli; extra == \"testing-integration\"", + "virtualenv>=13.0.0; extra == \"testing\"", + "virtualenv>=13.0.0; extra == \"testing-integration\"", + "wheel; extra == \"testing\"", + "wheel; extra == \"testing-integration\"" + ], + "requires_python": ">=3.7", + "version": "63.4.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", + "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "url": "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" + } + ], + "project_name": "six", + "requires_dists": [], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "1.16" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "url": "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", + "url": "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz" + } + ], + "project_name": "toml", + "requires_dists": [], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.6", + "version": "0.10.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f", + "url": "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz" + } + ], + "project_name": "tomli", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "2.0.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72", + "url": "https://files.pythonhosted.org/packages/d8/4e/db9505b53c44d7bc324a3d2e09bdf82b0943d6e08b183ae382860f482a87/typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c", + "url": "https://files.pythonhosted.org/packages/04/93/482d12fd3334b53ec4087e658ab161ab23affcf8b052166b4cf972ca673b/typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2", + "url": "https://files.pythonhosted.org/packages/07/d2/d55702e8deba2c80282fea0df53130790d8f398648be589750954c2dcce4/typed_ast-1.5.4.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97", + "url": "https://files.pythonhosted.org/packages/0b/e7/8ec06fc870254889198f933a595f139b7871b24bab1116d6128440731ea9/typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f", + "url": "https://files.pythonhosted.org/packages/2f/87/25abe9558ed6cbd83ad5bfdccf7210a7eefaaf0232f86de99f65992e91fd/typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3", + "url": "https://files.pythonhosted.org/packages/2f/d5/02059fe6ca70b11bb831007962323160372ca83843e0bf296e8b6d833198/typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6", + "url": "https://files.pythonhosted.org/packages/34/2d/17fc1845dd5210345904b054c9fa90f451d64df56de0470f429bc8d63d39/typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6", + "url": "https://files.pythonhosted.org/packages/40/1a/5731a1a3908f60032aead10c2ffc9af12ee708bc9a156ed14a5065a9873a/typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc", + "url": "https://files.pythonhosted.org/packages/78/18/3ecf5043f227ebd4a43af57e18e6a38f9fe0b81dbfbb8d62eec669d7b69e/typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d", + "url": "https://files.pythonhosted.org/packages/9b/d5/5540eb496c6817eaee8120fb759c7adb36f91ef647c6bb2877f09acc0569/typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66", + "url": "https://files.pythonhosted.org/packages/dd/87/09764c19a60a192b935579c93a07e781f6a52def10b723c8c5748e69a863/typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35", + "url": "https://files.pythonhosted.org/packages/f9/57/89ac0020d5ffc762487376d0c78e5d02af795657f18c411155b73de3c765/typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl" + } + ], + "project_name": "typed-ast", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "1.5.4" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", + "url": "https://files.pythonhosted.org/packages/ed/d6/2afc375a8d55b8be879d6b4986d4f69f01115e795e36827fd3a40166028b/typing_extensions-4.3.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6", + "url": "https://files.pythonhosted.org/packages/9e/1d/d128169ff58c501059330f1ad96ed62b79114a2eb30b8238af63a2e27f70/typing_extensions-4.3.0.tar.gz" + } + ], + "project_name": "typing-extensions", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "4.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "url": "https://files.pythonhosted.org/packages/d1/cb/4783c8f1a90f89e260dbf72ebbcf25931f3a28f8f80e2e90f8a589941b19/urllib3-1.26.11-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a", + "url": "https://files.pythonhosted.org/packages/6d/d5/e8258b334c9eb8eb78e31be92ea0d5da83ddd9385dc967dd92737604d239/urllib3-1.26.11.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "PySocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "brotli>=1.0.9; ((os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation == \"CPython\") and extra == \"brotli\"", + "brotlicffi>=0.8.0; ((os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\") and extra == \"brotli\"", + "brotlipy>=0.6.0; (os_name == \"nt\" and python_version < \"3\") and extra == \"brotli\"", + "certifi; extra == \"secure\"", + "cryptography>=1.3.4; extra == \"secure\"", + "idna>=2.0.0; extra == \"secure\"", + "ipaddress; python_version == \"2.7\" and extra == \"secure\"", + "pyOpenSSL>=0.14; extra == \"secure\"" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,<4,>=2.7", + "version": "1.26.11" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "5d55652dc1d0b3c734f044337d929aaf83f4f9138816ec680c1aefefb4dc4877", + "url": "https://files.pythonhosted.org/packages/67/b4/91683d7d5f66393e8877492fe4763304f82dbe308658a8db98f7a9e20baf/websocket_client-1.3.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "d58c5f284d6a9bf8379dab423259fe8f85b70d5fa5d2916d5791a84594b122b1", + "url": "https://files.pythonhosted.org/packages/0e/e7/e705ead133d21de4be752af4b3a0cb1f02514ff45bf165b3955c1ce22077/websocket-client-1.3.3.tar.gz" + } + ], + "project_name": "websocket-client", + "requires_dists": [ + "Sphinx>=3.4; extra == \"docs\"", + "python-socks; extra == \"optional\"", + "sphinx-rtd-theme>=0.5; extra == \"docs\"", + "websockets; extra == \"test\"", + "wsaccel; extra == \"optional\"" + ], + "requires_python": ">=3.7", + "version": "1.3.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009", + "url": "https://files.pythonhosted.org/packages/f0/36/639d6742bcc3ffdce8b85c31d79fcfae7bb04b95f0e5c4c6f8b206a038cc/zipp-3.8.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2", + "url": "https://files.pythonhosted.org/packages/3b/e3/fb79a1ea5f3a7e9745f688855d3c673f2ef7921639a380ec76f7d4d83a85/zipp-3.8.1.tar.gz" + } + ], + "project_name": "zipp", + "requires_dists": [ + "func-timeout; extra == \"testing\"", + "jaraco.itertools; extra == \"testing\"", + "jaraco.packaging>=9; extra == \"docs\"", + "jaraco.tidelift>=1.4; extra == \"docs\"", + "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-checkdocs>=2.4; extra == \"testing\"", + "pytest-cov; extra == \"testing\"", + "pytest-enabler>=1.3; extra == \"testing\"", + "pytest-flake8; extra == \"testing\"", + "pytest-mypy>=0.9.1; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest>=6; extra == \"testing\"", + "rst.linker>=1.9; extra == \"docs\"", + "sphinx; extra == \"docs\"" + ], + "requires_python": ">=3.7", + "version": "3.8.1" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.102", + "prefer_older_binary": false, + "requirements": [ + "hikaru==0.11.0b" + ], + "requires_python": [ + "<3.10,>=3.7" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/src/python/pants/backend/helm/subsystems/k8s_parser.py b/src/python/pants/backend/helm/subsystems/k8s_parser.py new file mode 100644 index 000000000000..6ea31b7a37d4 --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/k8s_parser.py @@ -0,0 +1,151 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +import pkgutil +from dataclasses import dataclass +from pathlib import PurePath + +from pants.backend.helm.utils.yaml import YamlPath +from pants.backend.python.goals import lockfile +from pants.backend.python.goals.lockfile import GeneratePythonLockfile +from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules import pex +from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess +from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel +from pants.engine.engine_aware import EngineAwareParameter +from pants.engine.fs import CreateDigest, Digest, FileContent, FileEntry +from pants.engine.process import ProcessResult +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.docutil import git_url +from pants.util.logging import LogLevel +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + +_HELM_K8S_PARSER_SOURCE = "k8s_parser_main.py" +_HELM_K8S_PARSER_PACKAGE = "pants.backend.helm.subsystems" + + +class HelmKubeParserSubsystem(PythonToolRequirementsBase): + options_scope = "helm-k8s-parser" + help = "Used to perform modifications to the final output produced by Helm charts when they've been fully rendered." + + default_version = "hikaru==0.11.0b" + + register_interpreter_constraints = True + default_interpreter_constraints = ["CPython>=3.7,<3.10"] + + register_lockfile = True + default_lockfile_resource = (_HELM_K8S_PARSER_PACKAGE, "k8s_parser.lock") + default_lockfile_path = ( + f"src/python/{_HELM_K8S_PARSER_PACKAGE.replace('.', '/')}/k8s_parser.lock" + ) + default_lockfile_url = git_url(default_lockfile_path) + + +class HelmKubeParserLockfileSentinel(GenerateToolLockfileSentinel): + resolve_name = HelmKubeParserSubsystem.options_scope + + +@rule +def setup_k8s_parser_lockfile_request( + _: HelmKubeParserLockfileSentinel, + post_renderer: HelmKubeParserSubsystem, + python_setup: PythonSetup, +) -> GeneratePythonLockfile: + return GeneratePythonLockfile.from_tool( + post_renderer, use_pex=python_setup.generate_lockfiles_with_pex + ) + + +@dataclass(frozen=True) +class _HelmKubeParserTool: + pex: VenvPex + + +@rule +async def build_k8s_parser_tool(k8s_parser: HelmKubeParserSubsystem) -> _HelmKubeParserTool: + parser_sources = pkgutil.get_data(_HELM_K8S_PARSER_PACKAGE, _HELM_K8S_PARSER_SOURCE) + if not parser_sources: + raise ValueError( + f"Unable to find source to {_HELM_K8S_PARSER_SOURCE!r} in {_HELM_K8S_PARSER_PACKAGE}" + ) + + parser_file_content = FileContent( + path="__k8s_parser.py", content=parser_sources, is_executable=True + ) + parser_digest = await Get(Digest, CreateDigest([parser_file_content])) + + parser_pex = await Get( + VenvPex, + PexRequest, + k8s_parser.to_pex_request( + main=EntryPoint(PurePath(parser_file_content.path).stem), sources=parser_digest + ), + ) + return _HelmKubeParserTool(parser_pex) + + +@dataclass(frozen=True) +class ParseKubeManifestRequest(EngineAwareParameter): + file: FileEntry + + def debug_hint(self) -> str | None: + return self.file.path + + +@dataclass(frozen=True) +class ParsedKubeManifest: + filename: str + found_image_refs: tuple[tuple[int, YamlPath, str], ...] + + +@rule(desc="Parse Kubernetes resource manifest") +async def parse_kube_manifest( + request: ParseKubeManifestRequest, tool: _HelmKubeParserTool +) -> ParsedKubeManifest: + file_digest = await Get(Digest, CreateDigest([request.file])) + + result = await Get( + ProcessResult, + VenvPexProcess( + tool.pex, + argv=[request.file.path], + input_digest=file_digest, + description=f"Parsing Kubernetes manifest {request.file.path}", + level=LogLevel.DEBUG, + ), + ) + + output = result.stdout.decode("utf-8").splitlines() + image_refs: list[tuple[int, YamlPath, str]] = [] + for line in output: + parts = line.split(",") + if len(parts) != 3: + raise Exception( + softwrap( + f"""Unexpected output from k8s parser when parsing file {request.file.path}: + + {line} + """ + ) + ) + + image_refs.append((int(parts[0]), YamlPath.parse(parts[1]), parts[2])) + + return ParsedKubeManifest(filename=request.file.path, found_image_refs=tuple(image_refs)) + + +def rules(): + return [ + *collect_rules(), + *pex.rules(), + *lockfile.rules(), + UnionRule(GenerateToolLockfileSentinel, HelmKubeParserLockfileSentinel), + ] diff --git a/src/python/pants/backend/helm/subsystems/k8s_parser_main.py b/src/python/pants/backend/helm/subsystems/k8s_parser_main.py new file mode 100644 index 000000000000..1fdf08784067 --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/k8s_parser_main.py @@ -0,0 +1,36 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import sys + +from hikaru import load_full_yaml + + +def main(args: list[str]): + input_filename = args[0] + + found_image_refs: dict[tuple[int, str], str] = {} + + with open(input_filename, "r") as file: + parsed_docs = load_full_yaml(stream=file) + + for idx, doc in enumerate(parsed_docs): + entries = doc.find_by_name("image") + for entry in entries: + entry_value = doc.object_at_path(entry.path) + entry_path = "/".join(map(str, entry.path)) + found_image_refs[(idx, entry_path)] = str(entry_value) + + for (idx, path), value in found_image_refs.items(): + print(f"{idx},/{path},{value}") + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("ERROR: Missing file argument", file=sys.stderr) + print(f"Syntax: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + main(sys.argv[1:]) diff --git a/src/python/pants/backend/helm/subsystems/k8s_parser_test.py b/src/python/pants/backend/helm/subsystems/k8s_parser_test.py new file mode 100644 index 000000000000..e86ce1c91932 --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/k8s_parser_test.py @@ -0,0 +1,76 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent +from typing import cast + +import pytest + +from pants.backend.helm.subsystems import k8s_parser +from pants.backend.helm.subsystems.k8s_parser import ParsedKubeManifest, ParseKubeManifestRequest +from pants.backend.helm.testutil import K8S_POD_FILE +from pants.backend.helm.utils.yaml import YamlPath +from pants.engine.fs import CreateDigest, Digest, DigestEntries, FileContent, FileEntry +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *k8s_parser.rules(), + QueryRule(ParsedKubeManifest, (ParseKubeManifestRequest,)), + QueryRule(DigestEntries, (Digest,)), + ] + ) + rule_runner.set_options( + [], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + return rule_runner + + +def test_parser_can_run(rule_runner: RuleRunner) -> None: + file_digest = rule_runner.request( + Digest, [CreateDigest([FileContent("pod.yaml", K8S_POD_FILE.encode("utf-8"))])] + ) + file_entries = rule_runner.request(DigestEntries, [file_digest]) + + parsed_manifest = rule_runner.request( + ParsedKubeManifest, [ParseKubeManifestRequest(cast(FileEntry, file_entries[0]))] + ) + + expected_image_refs = [ + (0, YamlPath.parse("/spec/containers/0/image"), "busybox:1.28"), + (0, YamlPath.parse("/spec/initContainers/0/image"), "busybox:1.29"), + ] + + assert parsed_manifest.found_image_refs == tuple(expected_image_refs) + + +def test_parser_returns_no_image_refs(rule_runner: RuleRunner) -> None: + config_map_contents = dedent( + """\ + apiVersion: v1 + kind: ConfigMap + metadata: + name: foo + data: + key: value + """ + ) + + file_digest = rule_runner.request( + Digest, + [CreateDigest([FileContent("config_map.yaml", config_map_contents.encode("utf-8"))])], + ) + file_entries = rule_runner.request(DigestEntries, [file_digest]) + + parsed_manifest = rule_runner.request( + ParsedKubeManifest, [ParseKubeManifestRequest(cast(FileEntry, file_entries[0]))] + ) + + assert len(parsed_manifest.found_image_refs) == 0 diff --git a/src/python/pants/backend/helm/subsystems/post_renderer.lock b/src/python/pants/backend/helm/subsystems/post_renderer.lock new file mode 100644 index 000000000000..a8d24a32e766 --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/post_renderer.lock @@ -0,0 +1,134 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// build-support/bin/generate_all_lockfiles.sh +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 2, +// "valid_for_interpreter_constraints": [ +// "CPython<3.10,>=3.7" +// ], +// "generated_with_requirements": [ +// "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96", +// "yamlpath<3.7,>=3.6" +// ] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "9af3ec5d7f8065582f3aa841305465025d0afd26c5fb54e15b964e11838fc74f", + "url": "https://files.pythonhosted.org/packages/c7/75/729b63cd0de2316c8bb789ff2c557d9732a5aeb900c5539ae74db41ba562/ruamel.yaml-0.17.17-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "9751de4cbb57d4bfbf8fc394e125ed4a2f170fbff3dc3d78abf50be85924f8be", + "url": "https://files.pythonhosted.org/packages/4d/15/7fc04de02ca774342800c9adf1a8239703977c49c5deaadec1689ec85506/ruamel.yaml-0.17.17.tar.gz" + } + ], + "project_name": "ruamel-yaml", + "requires_dists": [ + "ruamel.yaml.clib>=0.1.2; platform_python_implementation == \"CPython\" and python_version < \"3.10\"", + "ruamel.yaml.jinja2>=0.2; extra == \"jinja2\"", + "ryd; extra == \"docs\"" + ], + "requires_python": ">=3", + "version": "0.17.17" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0", + "url": "https://files.pythonhosted.org/packages/3e/36/f1e3b5a0507662a66f156518457ffaf530c818f204467a5c532fc44056f9/ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84", + "url": "https://files.pythonhosted.org/packages/15/7c/e65492dc1c311655760fb20a9f2512f419403fcdc9ada6c63f44d7fe7062/ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233", + "url": "https://files.pythonhosted.org/packages/23/2e/79d684c6cfa50b593f47938fec86f7c5d0208e0ecd278eef2ff0e10889d3/ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd", + "url": "https://files.pythonhosted.org/packages/8b/25/08e5ad2431a028d0723ca5540b3af6a32f58f25e83c6dda4d0fcef7288a3/ruamel.yaml.clib-0.2.6.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99", + "url": "https://files.pythonhosted.org/packages/98/8a/ba37489b423916162b086b01c7c18001cf297350694180468e1698085c58/ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed", + "url": "https://files.pythonhosted.org/packages/ba/2c/076d00f31f9476ccad3a6a5446ee30c5f0921012d714c76f3111e29b06ab/ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd", + "url": "https://files.pythonhosted.org/packages/d1/17/630d1d28e0fc442115280f3928b8a2b78a47b5c75bb619d16bfc4d046a69/ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl" + } + ], + "project_name": "ruamel-yaml-clib", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "0.2.6" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "7e2962aea4191a32ba4cbcf9e3c6a6f1836eb46204a6414d676a54f2df467c45", + "url": "https://files.pythonhosted.org/packages/20/f5/3540aacd19a3873ce0806d9d5c7a829813fb93a3731ca904d9b6cc405398/yamlpath-3.6.4-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "b2a746d91226f05f5b8f181ab12747e0ad59ab2f40f2625adaf5c2b8c524b812", + "url": "https://files.pythonhosted.org/packages/4e/20/be9fed27786422f5fef31b18765dcc5fe0019a34e62d8c57be341b0f6f5b/yamlpath-3.6.4.tar.gz" + } + ], + "project_name": "yamlpath", + "requires_dists": [ + "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17,>=0.15.96" + ], + "requires_python": ">3.6.0", + "version": "3.6.4" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.99", + "prefer_older_binary": false, + "requirements": [ + "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96", + "yamlpath<3.7,>=3.6" + ], + "requires_python": [ + "<3.10,>=3.7" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/src/python/pants/backend/helm/subsystems/post_renderer.py b/src/python/pants/backend/helm/subsystems/post_renderer.py new file mode 100644 index 000000000000..bba6d69b75bf --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/post_renderer.py @@ -0,0 +1,224 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +import os +import pkgutil +from dataclasses import dataclass +from pathlib import PurePath +from textwrap import dedent +from typing import Any + +import yaml + +from pants.backend.helm.utils.yaml import FrozenYamlIndex +from pants.backend.python.goals import lockfile +from pants.backend.python.goals.lockfile import GeneratePythonLockfile +from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules import pex +from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess +from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel +from pants.core.util_rules.system_binaries import CatBinary +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.internals.native_engine import MergeDigests +from pants.engine.process import Process +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.docutil import git_url +from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel + +logger = logging.getLogger(__name__) + +_HELM_POSTRENDERER_SOURCE = "post_renderer_main.py" +_HELM_POSTRENDERER_PACKAGE = "pants.backend.helm.subsystems" + + +class HelmPostRendererSubsystem(PythonToolRequirementsBase): + options_scope = "helm-post-renderer" + help = "Used perform modifications to the final output produced by Helm charts when they've been fully rendered." + + default_version = "yamlpath>=3.6,<3.7" + default_extra_requirements = [ + "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21" + ] + + register_interpreter_constraints = True + default_interpreter_constraints = ["CPython>=3.7,<3.10"] + + register_lockfile = True + default_lockfile_resource = (_HELM_POSTRENDERER_PACKAGE, "post_renderer.lock") + default_lockfile_path = ( + f"src/python/{_HELM_POSTRENDERER_PACKAGE.replace('.', '/')}/post_renderer.lock" + ) + default_lockfile_url = git_url(default_lockfile_path) + + +class HelmPostRendererLockfileSentinel(GenerateToolLockfileSentinel): + resolve_name = HelmPostRendererSubsystem.options_scope + + +@rule +def setup_postrenderer_lockfile_request( + _: HelmPostRendererLockfileSentinel, + post_renderer: HelmPostRendererSubsystem, + python_setup: PythonSetup, +) -> GeneratePythonLockfile: + return GeneratePythonLockfile.from_tool( + post_renderer, use_pex=python_setup.generate_lockfiles_with_pex + ) + + +_HELM_POST_RENDERER_TOOL = "__pants_helm_post_renderer.py" + + +@dataclass(frozen=True) +class _HelmPostRendererTool: + pex: VenvPex + + +@rule(desc="Setup Helm post renderer binaries", level=LogLevel.DEBUG) +async def setup_post_renderer_tool( + post_renderer: HelmPostRendererSubsystem, +) -> _HelmPostRendererTool: + post_renderer_sources = pkgutil.get_data(_HELM_POSTRENDERER_PACKAGE, _HELM_POSTRENDERER_SOURCE) + if not post_renderer_sources: + raise ValueError( + f"Unable to find source to {_HELM_POSTRENDERER_SOURCE!r} in {_HELM_POSTRENDERER_PACKAGE}" + ) + + post_renderer_content = FileContent( + path=_HELM_POST_RENDERER_TOOL, content=post_renderer_sources, is_executable=True + ) + post_renderer_digest = await Get(Digest, CreateDigest([post_renderer_content])) + + post_renderer_pex = await Get( + VenvPex, + PexRequest, + post_renderer.to_pex_request( + main=EntryPoint(PurePath(post_renderer_content.path).stem), sources=post_renderer_digest + ), + ) + return _HelmPostRendererTool(post_renderer_pex) + + +HELM_POST_RENDERER_CFG_FILENAME = "post_renderer.cfg.yaml" +_HELM_POST_RENDERER_WRAPPER_SCRIPT = "post_renderer_wrapper.sh" + + +@dataclass(frozen=True) +class SetupHelmPostRenderer(EngineAwareParameter): + """Request for a post-renderer process that will perform a series of replacements in the + generated files.""" + + replacements: FrozenYamlIndex[str] + description_of_origin: str + + def debug_hint(self) -> str | None: + return self.description_of_origin + + +@dataclass(frozen=True) +class HelmPostRenderer(EngineAwareReturnType): + exe: str + digest: Digest + immutable_input_digests: FrozenDict[str, Digest] + env: FrozenDict[str, str] + append_only_caches: FrozenDict[str, str] + description_of_origin: str + + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + + def message(self) -> str | None: + return f"runnable {self.exe} for {self.description_of_origin} is ready." + + def metadata(self) -> dict[str, Any] | None: + return {"exe": self.exe, "env": self.env, "append_only_caches": self.append_only_caches} + + +@rule(desc="Configure Helm post-renderer", level=LogLevel.DEBUG) +async def setup_post_renderer_launcher( + request: SetupHelmPostRenderer, + post_renderer_tool: _HelmPostRendererTool, + cat_binary: CatBinary, +) -> HelmPostRenderer: + # Build post-renderer configuration file and create a digest containing it. + post_renderer_config = yaml.safe_dump( + request.replacements.to_json_dict(), explicit_start=True, sort_keys=True + ) + post_renderer_cfg_digest = await Get( + Digest, + CreateDigest( + [ + FileContent(HELM_POST_RENDERER_CFG_FILENAME, post_renderer_config.encode("utf-8")), + ] + ), + ) + + # Generate a temporary PEX process that uses the previously created configuration file. + post_renderer_cfg_file = os.path.join(".", HELM_POST_RENDERER_CFG_FILENAME) + post_renderer_stdin_file = os.path.join(".", "__stdin.yaml") + post_renderer_process = await Get( + Process, + VenvPexProcess( + post_renderer_tool.pex, + argv=[post_renderer_cfg_file, post_renderer_stdin_file], + input_digest=post_renderer_cfg_digest, + description="", + ), + ) + + # Build a shell wrapper script which will be the actual entry-point sent to Helm as the post-renderer. + post_renderer_process_cli = " ".join(post_renderer_process.argv) + logger.debug(f"Built post-renderer process CLI: {post_renderer_process_cli}") + + postrenderer_wrapper_script = dedent( + f"""\ + #!/bin/bash + + # Output stdin into a file in disk + {cat_binary.path} <&0 > {post_renderer_stdin_file} + + {post_renderer_process_cli} + """ + ) + wrapper_digest = await Get( + Digest, + CreateDigest( + [ + FileContent( + _HELM_POST_RENDERER_WRAPPER_SCRIPT, + postrenderer_wrapper_script.encode("utf-8"), + is_executable=True, + ), + ] + ), + ) + + # Extract all info needed to invoke the post-renderer from the PEX process + launcher_digest = await Get( + Digest, MergeDigests([wrapper_digest, post_renderer_process.input_digest]) + ) + return HelmPostRenderer( + exe=_HELM_POST_RENDERER_WRAPPER_SCRIPT, + digest=launcher_digest, + env=post_renderer_process.env, + append_only_caches=post_renderer_process.append_only_caches, + immutable_input_digests=post_renderer_process.immutable_input_digests, + description_of_origin=request.description_of_origin, + ) + + +def rules(): + return [ + *collect_rules(), + *pex.rules(), + *lockfile.rules(), + UnionRule(GenerateToolLockfileSentinel, HelmPostRendererLockfileSentinel), + ] diff --git a/src/python/pants/backend/helm/subsystems/post_renderer_main.py b/src/python/pants/backend/helm/subsystems/post_renderer_main.py new file mode 100644 index 000000000000..2edd87ae0dc0 --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/post_renderer_main.py @@ -0,0 +1,127 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import sys +from collections import defaultdict +from types import SimpleNamespace +from typing import Any + +from ruamel.yaml import YAML +from ruamel.yaml.compat import StringIO +from yamlpath import Processor # pants: no-infer-dep +from yamlpath.common import Parsers # pants: no-infer-dep +from yamlpath.exceptions import YAMLPathException # pants: no-infer-dep +from yamlpath.wrappers import ConsolePrinter # pants: no-infer-dep + +_SOURCE_FILENAME_PREFIX = "# Source: " + + +def build_manifest_map(input_file: str) -> dict[str, list[str]]: + """Parses the contents that are being received from Helm. + + Helm will send us input that follows the following format: + + ```yaml + --- + # Source: filename.yaml + data: + key: value + ``` + + Since there are cases in which the same source may produce more than + one YAML structure, the returned type represents this with a dictionary + of lists, in which the key is the source filename and each item in the list + is the content following the `# Source: ...` header. + """ + + result = defaultdict(list) + template_files = [] + with open(input_file, "r", encoding="utf-8") as f: + template_files = f.read().split("---") + + for template in template_files: + lines = [line for line in template.splitlines() if line and len(line) > 0] + if not lines: + continue + + template_name = lines[0][len(_SOURCE_FILENAME_PREFIX) :] + result[template_name].append("\n".join(lines[1:])) + + return result + + +def dump_yaml_data(yaml: YAML, data: Any) -> str: + stream = StringIO() + yaml.dump(data, stream) + return stream.getvalue() + + +def print_manifests(templates: dict[str, list[str]]) -> None: + """Outputs to standard out the contents of the different manifests following the same format + used by Helm when sending them into us.""" + + if not templates: + return + + for filename, documents in templates.items(): + for content in documents: + print("---") + print(f"{_SOURCE_FILENAME_PREFIX}{filename}") + print(content) + + +def main(args: list[str]) -> None: + cfg_file = args[0] + manifests_stdin_file = args[1] + + logging_args = SimpleNamespace(quiet=True, verbose=False, debug=False) + log = ConsolePrinter(logging_args) + + yaml = Parsers.get_yaml_editor(explicit_start=False, preserve_quotes=False) + + input_manifest_map = build_manifest_map(manifests_stdin_file) + output_manifest_map: dict[str, list[str]] = defaultdict(list) + + # `cfg_yaml` is the data structure parsed from the YAML index built while preparing this + # post-renderer instance. + (cfg_yaml, doc_loaded) = Parsers.get_yaml_data(yaml, log, cfg_file) + if not doc_loaded: + exit(1) + + # Go through the items in the configuration file and apply the replacements requested. + for source_filename, source_changes_list in cfg_yaml.items(): + input_manifests = input_manifest_map.get(source_filename) + if not input_manifests: + continue + + for input_manifest, manifest_change_spec in zip(input_manifests, source_changes_list): + manifest_change_paths = manifest_change_spec["paths"] + + # Manifests that require no changes will have an empty `paths` element in the change spec, + # so we add to the output map the manifest document unchanged. + if not manifest_change_paths: + output_manifest_map[source_filename].append(input_manifest) + continue + + (manifest_yaml, doc_loaded) = Parsers.get_yaml_data( + yaml, log, input_manifest, literal=True + ) + if not doc_loaded: + continue + + processor = Processor(log, manifest_yaml) + for path_spec, replacement in manifest_change_paths.items(): + try: + processor.set_value(path_spec, replacement) + except YAMLPathException as ex: + log.critical(ex, 119) + + output_manifest_map[source_filename].append(dump_yaml_data(yaml, manifest_yaml)) + + print_manifests(output_manifest_map) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/src/python/pants/backend/helm/subsystems/post_renderer_test.py b/src/python/pants/backend/helm/subsystems/post_renderer_test.py new file mode 100644 index 000000000000..ba742e951b5a --- /dev/null +++ b/src/python/pants/backend/helm/subsystems/post_renderer_test.py @@ -0,0 +1,76 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from pathlib import PurePath +from textwrap import dedent + +import pytest + +from pants.backend.helm.subsystems import post_renderer +from pants.backend.helm.subsystems.post_renderer import HelmPostRenderer, SetupHelmPostRenderer +from pants.backend.helm.utils.yaml import MutableYamlIndex, YamlPath +from pants.engine.fs import DigestContents, Snapshot +from pants.engine.process import Process, ProcessResult +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *post_renderer.rules(), + QueryRule(HelmPostRenderer, (SetupHelmPostRenderer,)), + QueryRule(ProcessResult, (Process,)), + ] + ) + rule_runner.set_options( + [], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + return rule_runner + + +def test_post_renderer_is_runnable(rule_runner: RuleRunner) -> None: + replacements = MutableYamlIndex[str]() + replacements.insert( + file_path=PurePath("file.yaml"), + yaml_path=YamlPath.parse("/root/element"), + item="replaced_value", + ) + + expected_cfg_file = dedent( + """\ + --- + file.yaml: + - paths: + /root/element: replaced_value + """ + ) + + post_renderer_setup = rule_runner.request( + HelmPostRenderer, + [ + SetupHelmPostRenderer( + replacements.frozen(), description_of_origin="test_post_renderer_is_runnable" + ) + ], + ) + assert post_renderer_setup.exe == "post_renderer_wrapper.sh" + + input_snapshot = rule_runner.request(Snapshot, [post_renderer_setup.digest]) + assert "post_renderer.cfg.yaml" in input_snapshot.files + assert "post_renderer_wrapper.sh" in input_snapshot.files + + input_contents = rule_runner.request(DigestContents, [post_renderer_setup.digest]) + for file in input_contents: + if file.path == "post_renderer.cfg.yaml": + assert file.content.decode() == expected_cfg_file + elif file.path == "post_renderer_wrapper.sh": + script_lines = file.content.decode().splitlines() + assert ( + "./helm_post_renderer.pex_pex_shim.sh ./post_renderer.cfg.yaml ./__stdin.yaml" + in script_lines + ) diff --git a/src/python/pants/backend/helm/subsystems/unittest.py b/src/python/pants/backend/helm/subsystems/unittest.py index 5c0bcab02863..6c86c90d7f8c 100644 --- a/src/python/pants/backend/helm/subsystems/unittest.py +++ b/src/python/pants/backend/helm/subsystems/unittest.py @@ -3,7 +3,7 @@ from enum import Enum -from pants.backend.helm.util_rules.plugins import ( +from pants.backend.helm.util_rules.tool import ( ExternalHelmPlugin, ExternalHelmPluginBinding, ExternalHelmPluginRequest, @@ -62,7 +62,7 @@ class HelmUnitTestPluginBinding(ExternalHelmPluginBinding[HelmUnitTestSubsystem] @rule -def download_plugin_request( +def download_unittest_plugin_request( _: HelmUnitTestPluginBinding, subsystem: HelmUnitTestSubsystem ) -> ExternalHelmPluginRequest: return ExternalHelmPluginRequest.from_subsystem(subsystem) diff --git a/src/python/pants/backend/helm/subsystems/unittest_test.py b/src/python/pants/backend/helm/subsystems/unittest_test.py index b4e22c3e0262..58fc615ebcc3 100644 --- a/src/python/pants/backend/helm/subsystems/unittest_test.py +++ b/src/python/pants/backend/helm/subsystems/unittest_test.py @@ -7,12 +7,10 @@ import pytest -from pants.backend.helm.subsystems import unittest +from pants.backend.helm.subsystems import unittest as unittest_subsystem from pants.backend.helm.subsystems.unittest import HelmUnitTestSubsystem from pants.backend.helm.util_rules import tool from pants.backend.helm.util_rules.tool import HelmProcess -from pants.core.util_rules import external_tool -from pants.engine import process from pants.engine.fs import EMPTY_DIGEST from pants.engine.process import ProcessResult from pants.engine.rules import QueryRule @@ -23,10 +21,8 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ - *external_tool.rules(), *tool.rules(), - *process.rules(), - *unittest.rules(), + *unittest_subsystem.rules(), QueryRule(ProcessResult, (HelmProcess,)), ] ) diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index 8fb65685f0cc..e90b6871ea7b 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -14,7 +14,10 @@ AllTargets, BoolField, Dependencies, + DescriptionField, + DictStringToStringField, FieldSet, + IntField, MultipleSourcesField, OverridesField, SingleSourceField, @@ -24,6 +27,7 @@ TargetFilesGenerator, Targets, TriBoolField, + ValidNumbers, generate_file_based_overrides_field_help_message, generate_multiple_sources_field_help_message, ) @@ -289,8 +293,6 @@ class HelmUnitTestTestsGeneratorTarget(TargetFilesGenerator): *COMMON_TARGET_FIELDS, HelmUnitTestGeneratingSourcesField, HelmUnitTestDependenciesField, - HelmUnitTestStrictField, - HelmUnitTestTimeoutField, HelmUnitTestOverridesField, ) generated_target_cls = HelmUnitTestTestTarget @@ -373,5 +375,107 @@ def all_helm_artifact_targets(all_targets: AllTargets) -> AllHelmArtifactTargets ) +# ----------------------------------------------------------------------------------------------- +# `helm_deployment` target +# ----------------------------------------------------------------------------------------------- + + +class HelmDeploymentReleaseNameField(StringField): + alias = "release_name" + help = "Name of the release used in the deployment. If not set, the target name will be used instead." + + +class HelmDeploymentNamespaceField(StringField): + alias = "namespace" + help = "Kubernetes namespace for the given deployment." + + +class HelmDeploymentDependenciesField(Dependencies): + pass + + +class HelmDeploymentSkipCrdsField(BoolField): + alias = "skip_crds" + default = False + help = "If true, then does not deploy the Custom Resource Definitions that are defined in the chart." + + +class HelmDeploymentSourcesField(MultipleSourcesField): + default = ("*.yaml", "*.yml") + expected_file_extensions = (".yaml", ".yml") + help = "Helm configuration files for a given deployment." + + +class HelmDeploymentValuesField(DictStringToStringField): + alias = "values" + required = False + help = "Individual values to use when rendering a given deployment." + + +class HelmDeploymentCreateNamespaceField(BoolField): + alias = "create_namespace" + default = False + help = "If true, the namespace will be created if it doesn't exist." + + +class HelmDeploymentNoHooksField(BoolField): + alias = "no_hooks" + default = False + help = "If true, none of the lifecycle hooks of the given chart will be included in the deployment." + + +class HelmDeploymentTimeoutField(IntField): + alias = "timeout" + required = False + help = "Timeout in seconds when running a Helm deployment." + valid_numbers = ValidNumbers.positive_only + + +class HelmDeploymentTarget(Target): + alias = "helm_deployment" + core_fields = ( + *COMMON_TARGET_FIELDS, + HelmDeploymentReleaseNameField, + HelmDeploymentDependenciesField, + HelmDeploymentSourcesField, + HelmDeploymentNamespaceField, + HelmDeploymentSkipCrdsField, + HelmDeploymentValuesField, + HelmDeploymentCreateNamespaceField, + HelmDeploymentNoHooksField, + HelmDeploymentTimeoutField, + ) + help = "A Helm chart deployment." + + +@dataclass(frozen=True) +class HelmDeploymentFieldSet(FieldSet): + required_fields = ( + HelmDeploymentDependenciesField, + HelmDeploymentSourcesField, + ) + + description: DescriptionField + release_name: HelmDeploymentReleaseNameField + namespace: HelmDeploymentNamespaceField + create_namespace: HelmDeploymentCreateNamespaceField + sources: HelmDeploymentSourcesField + skip_crds: HelmDeploymentSkipCrdsField + no_hooks: HelmDeploymentNoHooksField + dependencies: HelmDeploymentDependenciesField + values: HelmDeploymentValuesField + + +class AllHelmDeploymentTargets(Targets): + pass + + +@rule +def all_helm_deployment_targets(targets: AllTargets) -> AllHelmDeploymentTargets: + return AllHelmDeploymentTargets( + [tgt for tgt in targets if HelmDeploymentFieldSet.is_applicable(tgt)] + ) + + def rules(): return collect_rules() diff --git a/src/python/pants/backend/helm/target_types_test.py b/src/python/pants/backend/helm/target_types_test.py index 4b7d20a7c959..314e5a8338f2 100644 --- a/src/python/pants/backend/helm/target_types_test.py +++ b/src/python/pants/backend/helm/target_types_test.py @@ -15,7 +15,7 @@ HELM_CHART_FILE, HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, ) from pants.engine.addresses import Address from pants.engine.internals.graph import _TargetParametrizations, _TargetParametrizationsRequest @@ -39,7 +39,7 @@ def test_generate_source_targets() -> None: f"{source_root}/Chart.yaml": HELM_CHART_FILE, f"{source_root}/values.yaml": HELM_VALUES_FILE, f"{source_root}/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - f"{source_root}/templates/service.yaml": K8S_SERVICE_FILE, + f"{source_root}/templates/service.yaml": K8S_SERVICE_TEMPLATE, f"{source_root}/tests/BUILD": "helm_unittest_tests(name='foo_tests')", f"{source_root}/tests/service_test.yaml": "", } diff --git a/src/python/pants/backend/helm/test/unittest.py b/src/python/pants/backend/helm/test/unittest.py index c62bc4174a05..134f220b3976 100644 --- a/src/python/pants/backend/helm/test/unittest.py +++ b/src/python/pants/backend/helm/test/unittest.py @@ -18,6 +18,7 @@ HelmUnitTestTestTarget, HelmUnitTestTimeoutField, ) +from pants.backend.helm.util_rules import tool from pants.backend.helm.util_rules.chart import HelmChart, HelmChartRequest from pants.backend.helm.util_rules.tool import HelmProcess from pants.core.goals.test import ( @@ -32,9 +33,8 @@ from pants.core.util_rules.stripped_source_files import StrippedSourceFiles from pants.engine.addresses import Address from pants.engine.fs import AddPrefix, Digest, MergeDigests, RemovePrefix, Snapshot -from pants.engine.internals.selectors import MultiGet from pants.engine.process import FallibleProcessResult, ProcessCacheScope -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( DependenciesRequest, SourcesField, @@ -168,5 +168,6 @@ def rules(): *collect_rules(), *subsystem_rules(), *dependency_rules(), + *tool.rules(), UnionRule(TestFieldSet, HelmUnitTestFieldSet), ] diff --git a/src/python/pants/backend/helm/test/unittest_test.py b/src/python/pants/backend/helm/test/unittest_test.py index 9ace461933e0..6efb190e9429 100644 --- a/src/python/pants/backend/helm/test/unittest_test.py +++ b/src/python/pants/backend/helm/test/unittest_test.py @@ -8,14 +8,14 @@ from pants.backend.helm.target_types import HelmChartTarget, HelmUnitTestTestTarget from pants.backend.helm.target_types import rules as target_types_rules from pants.backend.helm.test.unittest import HelmUnitTestFieldSet -from pants.backend.helm.test.unittest import rules as test_rules +from pants.backend.helm.test.unittest import rules as unittest_rules from pants.backend.helm.testutil import ( HELM_CHART_FILE, HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, ) -from pants.backend.helm.util_rules import chart, tool +from pants.backend.helm.util_rules import chart from pants.core.goals.test import TestResult from pants.core.util_rules import external_tool, stripped_source_files from pants.engine.addresses import Address @@ -30,9 +30,8 @@ def rule_runner() -> RuleRunner: target_types=[HelmChartTarget, HelmUnitTestTestTarget], rules=[ *external_tool.rules(), - *tool.rules(), *chart.rules(), - *test_rules(), + *unittest_rules(), *stripped_source_files.rules(), *source_root_rules(), *target_types_rules(), @@ -48,7 +47,7 @@ def test_simple_success(rule_runner: RuleRunner) -> None: "Chart.yaml": HELM_CHART_FILE, "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, "tests/BUILD": "helm_unittest_test(name='test', source='service_test.yaml')", "tests/service_test.yaml": dedent( """\ @@ -87,7 +86,7 @@ def test_simple_failure(rule_runner: RuleRunner) -> None: "Chart.yaml": HELM_CHART_FILE, "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, "tests/BUILD": "helm_unittest_test(name='test', source='service_test.yaml')", "tests/service_test.yaml": dedent( """\ diff --git a/src/python/pants/backend/helm/testutil.py b/src/python/pants/backend/helm/testutil.py index 1a49fee823dd..b20069631d76 100644 --- a/src/python/pants/backend/helm/testutil.py +++ b/src/python/pants/backend/helm/testutil.py @@ -123,7 +123,7 @@ def gen_chart_file( """ ) -K8S_SERVICE_FILE = dedent( +K8S_SERVICE_TEMPLATE = dedent( """\ apiVersion: v1 kind: Service @@ -143,7 +143,7 @@ def gen_chart_file( """ ) -K8S_INGRESS_FILE_WITH_LINT_WARNINGS = dedent( +K8S_INGRESS_TEMPLATE_WITH_LINT_WARNINGS = dedent( """\ apiVersion: extensions/v1beta1 kind: Ingress @@ -166,22 +166,40 @@ def gen_chart_file( """ ) +K8S_POD_TEMPLATE = dedent( + """\ + apiVersion: v1 + kind: Pod + metadata: + name: {{ template "fullname" . }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + spec: + containers: + - name: myapp-container + image: busybox:1.28 + initContainers: + - name: init-service + image: busybox:1.29 + """ +) + K8S_POD_FILE = dedent( """\ - apiVersion: v1 - kind: Pod - metadata: - name: {{ template "fullname" . }} - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - spec: - containers: - - name: myapp-container - image: busybox:1.28 - initContainers: - - name: init-service - image: busybox:1.29 - """ + apiVersion: v1 + kind: Pod + metadata: + name: foo + labels: + chart: foo-bar + spec: + containers: + - name: myapp-container + image: busybox:1.28 + initContainers: + - name: init-service + image: busybox:1.29 + """ ) K8S_CRD_FILE = dedent( @@ -278,3 +296,37 @@ def gen_chart_file( internalPort: 1223 """ ) + +HELM_BATCH_HOOK_TEMPLATE = dedent( + """\ + apiVersion: batch/v1 + kind: Job + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-install + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded + spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + containers: + - name: post-install-job + image: "alpine:3.3" + command: ["/bin/sleep","{{ default "10" .Values.sleepyTime }}"] + """ +) diff --git a/src/python/pants/backend/helm/util_rules/chart.py b/src/python/pants/backend/helm/util_rules/chart.py index 9db6a6f7b1da..85bd001a6dae 100644 --- a/src/python/pants/backend/helm/util_rules/chart.py +++ b/src/python/pants/backend/helm/util_rules/chart.py @@ -16,7 +16,12 @@ FetchHelmArfifactsRequest, ) from pants.backend.helm.subsystems.helm import HelmSubsystem -from pants.backend.helm.target_types import HelmChartFieldSet, HelmChartMetaSourceField +from pants.backend.helm.target_types import ( + HelmChartFieldSet, + HelmChartMetaSourceField, + HelmChartTarget, + HelmDeploymentFieldSet, +) from pants.backend.helm.util_rules import chart_metadata, sources from pants.backend.helm.util_rules.chart_metadata import ( HELM_CHART_METADATA_FILENAMES, @@ -25,7 +30,8 @@ ParseHelmChartMetadataDigest, ) from pants.backend.helm.util_rules.sources import HelmChartSourceFiles, HelmChartSourceFilesRequest -from pants.engine.addresses import Address +from pants.engine.addresses import Address, Addresses +from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( EMPTY_DIGEST, AddPrefix, @@ -36,34 +42,44 @@ Snapshot, ) from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import DependenciesRequest, Target, Targets +from pants.engine.target import DependenciesRequest, ExplicitlyProvidedDependencies, Target, Targets from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet -from pants.util.strutil import pluralize +from pants.util.strutil import pluralize, softwrap logger = logging.getLogger(__name__) +class InvalidHelmChartTarget(ValueError): + def __init__(self, target: Target) -> None: + super().__init__(f"The target {target.address} is not a `{HelmChartTarget.alias}`.") + + @dataclass(frozen=True) class HelmChart: address: Address - metadata: HelmChartMetadata + info: HelmChartMetadata snapshot: Snapshot artifact: ResolvedHelmArtifact | None = None @property def path(self) -> str: - return self.metadata.name + return self.info.name @dataclass(frozen=True) -class HelmChartRequest: +class HelmChartRequest(EngineAwareParameter): field_set: HelmChartFieldSet @classmethod def from_target(cls, target: Target) -> HelmChartRequest: + if not HelmChartFieldSet.is_applicable(target): + raise InvalidHelmChartTarget(target) return cls(HelmChartFieldSet.create(target)) + def debug_hint(self) -> str | None: + return self.field_set.address.spec + @rule async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> HelmChart: @@ -85,7 +101,7 @@ async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> H @rule(desc="Collect all source code and subcharts of a Helm Chart", level=LogLevel.DEBUG) async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> HelmChart: - dependencies, source_files, metadata = await MultiGet( + dependencies, source_files, chart_info = await MultiGet( Get(Targets, DependenciesRequest(request.field_set.dependencies)), Get( HelmChartSourceFiles, @@ -121,7 +137,12 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> subcharts_digest = EMPTY_DIGEST if subcharts: logger.debug( - f"Found {pluralize(len(subcharts), 'subchart')} as direct dependencies on Helm chart at: {request.field_set.address}" + softwrap( + f""" + Found {pluralize(len(subcharts), 'subchart')} as direct dependencies + on Helm chart at: {request.field_set.address}. + """ + ) ) merged_subcharts = await Get( @@ -131,9 +152,9 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> # Update subchart dependencies in the metadata and re-render it. remotes = subsystem.remotes() - subchart_map: dict[str, HelmChart] = {chart.metadata.name: chart for chart in subcharts} + subchart_map: dict[str, HelmChart] = {chart.info.name: chart for chart in subcharts} updated_dependencies: OrderedSet[HelmChartDependency] = OrderedSet() - for dep in metadata.dependencies: + for dep in chart_info.dependencies: updated_dep = dep if not dep.repository and remotes.default_registry: @@ -146,7 +167,7 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> if dep.name in subchart_map: updated_dep = dataclasses.replace( - updated_dep, version=subchart_map[dep.name].metadata.version + updated_dep, version=subchart_map[dep.name].info.version ) updated_dependencies.add(updated_dep) @@ -154,7 +175,7 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> # Include the explicitly provided subchats in the set of dependencies if not already present. updated_dependencies_names = {dep.name for dep in updated_dependencies} remaining_subcharts = [ - chart for chart in subcharts if chart.metadata.name not in updated_dependencies_names + chart for chart in subcharts if chart.info.name not in updated_dependencies_names ] for chart in remaining_subcharts: if chart.artifact: @@ -164,17 +185,15 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> repository=chart.artifact.location_url, ) else: - dependency = HelmChartDependency( - name=chart.metadata.name, version=chart.metadata.version - ) + dependency = HelmChartDependency(name=chart.info.name, version=chart.info.version) updated_dependencies.add(dependency) # Update metadata with the information about charts' dependencies. - metadata = dataclasses.replace(metadata, dependencies=tuple(updated_dependencies)) + chart_info = dataclasses.replace(chart_info, dependencies=tuple(updated_dependencies)) # Re-render the Chart.yaml file with the updated dependencies. metadata_digest, sources_without_metadata = await MultiGet( - Get(Digest, HelmChartMetadata, metadata), + Get(Digest, HelmChartMetadata, chart_info), Get( Digest, DigestSubset( @@ -191,8 +210,56 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> Digest, MergeDigests([metadata_digest, sources_without_metadata, subcharts_digest]) ) - chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, metadata.name)) - return HelmChart(address=request.field_set.address, metadata=metadata, snapshot=chart_snapshot) + chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, chart_info.name)) + return HelmChart(address=request.field_set.address, info=chart_info, snapshot=chart_snapshot) + + +class MissingHelmDeploymentChartError(ValueError): + def __init__(self, address: Address) -> None: + super().__init__( + f"The target '{address}' is missing a dependency on a `{HelmChartTarget.alias}` target." + ) + + +class TooManyChartDependenciesError(ValueError): + def __init__(self, address: Address) -> None: + super().__init__( + f"The target '{address}' has too many `{HelmChartTarget.alias}` " + "addresses in its dependencies, it should have only one." + ) + + +@dataclass(frozen=True) +class FindHelmDeploymentChart(EngineAwareParameter): + field_set: HelmDeploymentFieldSet + + def debug_hint(self) -> str | None: + return self.field_set.address.spec + + +@rule(desc="Find Helm deployment's chart", level=LogLevel.DEBUG) +async def find_chart_for_deployment(request: FindHelmDeploymentChart) -> HelmChartRequest: + explicit_dependencies = await Get( + ExplicitlyProvidedDependencies, DependenciesRequest(request.field_set.dependencies) + ) + explicit_targets = await Get( + Targets, + Addresses( + [ + addr + for addr in explicit_dependencies.includes + if addr not in explicit_dependencies.ignores + ] + ), + ) + + found_charts = [tgt for tgt in explicit_targets if HelmChartFieldSet.is_applicable(tgt)] + if not found_charts: + raise MissingHelmDeploymentChartError(request.field_set.address) + if len(found_charts) > 1: + raise TooManyChartDependenciesError(request.field_set.address) + + return HelmChartRequest.from_target(found_charts[0]) def rules(): diff --git a/src/python/pants/backend/helm/util_rules/chart_metadata.py b/src/python/pants/backend/helm/util_rules/chart_metadata.py index 684320132d01..aed5801feeaa 100644 --- a/src/python/pants/backend/helm/util_rules/chart_metadata.py +++ b/src/python/pants/backend/helm/util_rules/chart_metadata.py @@ -12,7 +12,7 @@ import yaml from pants.backend.helm.target_types import HelmChartMetaSourceField -from pants.backend.helm.util_rules.yaml_utils import snake_case_attr_dict +from pants.backend.helm.utils.yaml import snake_case_attr_dict from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.engine.fs import ( CreateDigest, diff --git a/src/python/pants/backend/helm/util_rules/chart_test.py b/src/python/pants/backend/helm/util_rules/chart_test.py index 4713c1df21bd..c0fe9e18f171 100644 --- a/src/python/pants/backend/helm/util_rules/chart_test.py +++ b/src/python/pants/backend/helm/util_rules/chart_test.py @@ -7,16 +7,21 @@ import pytest -from pants.backend.helm.target_types import HelmArtifactTarget, HelmChartTarget +from pants.backend.helm.target_types import ( + HelmArtifactTarget, + HelmChartTarget, + HelmDeploymentFieldSet, + HelmDeploymentTarget, +) from pants.backend.helm.target_types import rules as target_types_rules from pants.backend.helm.testutil import ( HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, gen_chart_file, ) -from pants.backend.helm.util_rules import chart, sources, tool -from pants.backend.helm.util_rules.chart import HelmChart, HelmChartRequest +from pants.backend.helm.util_rules import chart +from pants.backend.helm.util_rules.chart import FindHelmDeploymentChart, HelmChart, HelmChartRequest from pants.backend.helm.util_rules.chart_metadata import ( ChartType, HelmChartDependency, @@ -24,8 +29,7 @@ ParseHelmChartMetadataDigest, ) from pants.build_graph.address import Address -from pants.core.util_rules import config_files, external_tool, stripped_source_files -from pants.engine import process +from pants.engine.internals.scheduler import ExecutionError from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -33,18 +37,13 @@ @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - target_types=[HelmChartTarget, HelmArtifactTarget], + target_types=[HelmChartTarget, HelmArtifactTarget, HelmDeploymentTarget], rules=[ - *config_files.rules(), - *external_tool.rules(), *chart.rules(), - *sources.rules(), - *tool.rules(), - *process.rules(), - *stripped_source_files.rules(), *target_types_rules(), QueryRule(HelmChart, (HelmChartRequest,)), QueryRule(HelmChartMetadata, (ParseHelmChartMetadataDigest,)), + QueryRule(HelmChart, (FindHelmDeploymentChart,)), ], ) @@ -69,7 +68,7 @@ def test_collects_single_chart_sources( "Chart.yaml": gen_chart_file(name, version=version, type=type, icon=icon), "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, } ) @@ -85,7 +84,7 @@ def test_collects_single_chart_sources( helm_chart = rule_runner.request(HelmChart, [HelmChartRequest.from_target(tgt)]) assert not helm_chart.artifact - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert len(helm_chart.snapshot.files) == 4 assert helm_chart.address == address @@ -103,7 +102,7 @@ def test_gathers_local_subchart_sources_using_explicit_dependency(rule_runner: R ), "src/chart1/values.yaml": HELM_VALUES_FILE, "src/chart1/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "src/chart1/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart1/templates/service.yaml": K8S_SERVICE_TEMPLATE, "src/chart2/BUILD": "helm_chart(dependencies=['//src/chart1'])", "src/chart2/Chart.yaml": dedent( """\ @@ -126,9 +125,9 @@ def test_gathers_local_subchart_sources_using_explicit_dependency(rule_runner: R assert "chart2/charts/chart1" in helm_chart.snapshot.dirs assert "chart2/charts/chart1/templates/service.yaml" in helm_chart.snapshot.files - assert len(helm_chart.metadata.dependencies) == 1 - assert helm_chart.metadata.dependencies[0].name == "chart1" - assert helm_chart.metadata.dependencies[0].alias == "foo" + assert len(helm_chart.info.dependencies) == 1 + assert helm_chart.info.dependencies[0].name == "chart1" + assert helm_chart.info.dependencies[0].alias == "foo" def test_gathers_all_subchart_sources_inferring_dependencies(rule_runner: RuleRunner) -> None: @@ -154,7 +153,7 @@ def test_gathers_all_subchart_sources_inferring_dependencies(rule_runner: RuleRu ), "src/chart1/values.yaml": HELM_VALUES_FILE, "src/chart1/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "src/chart1/templates/service.yaml": K8S_SERVICE_FILE, + "src/chart1/templates/service.yaml": K8S_SERVICE_TEMPLATE, "src/chart2/BUILD": "helm_chart()", "src/chart2/Chart.yaml": dedent( """\ @@ -197,7 +196,7 @@ def test_gathers_all_subchart_sources_inferring_dependencies(rule_runner: RuleRu target = rule_runner.get_target(Address("src/chart2", target_name="chart2")) helm_chart = rule_runner.request(HelmChart, [HelmChartRequest.from_target(target)]) - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert "chart2/charts/chart1" in helm_chart.snapshot.dirs assert "chart2/charts/chart1/templates/service.yaml" in helm_chart.snapshot.files assert "chart2/charts/cert-manager" in helm_chart.snapshot.dirs @@ -275,5 +274,70 @@ def test_chart_metadata_is_updated_with_explicit_dependencies(rule_runner: RuleR ], ) - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert new_metadata == expected_metadata + + +def test_obtain_chart_from_deployment(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/foo/BUILD": "helm_chart()", + "src/foo/Chart.yaml": gen_chart_file("foo", version="1.0.0"), + "src/bar/BUILD": dedent( + """\ + helm_deployment(dependencies=["//src/foo"]) + """ + ), + } + ) + + source_root_patterns = ("/src/*",) + rule_runner.set_options([f"--source-root-patterns={repr(source_root_patterns)}"]) + + target = rule_runner.get_target(Address("src/bar")) + field_set = HelmDeploymentFieldSet.create(target) + + chart = rule_runner.request(HelmChart, [FindHelmDeploymentChart(field_set)]) + + assert chart.info.name == "foo" + assert chart.info.version == "1.0.0" + + +def test_fail_when_no_chart_dependency_is_found_for_a_deployment(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"BUILD": """helm_deployment(name="foo")"""}) + + target = rule_runner.get_target(Address("", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(target) + + msg = f"The target '{field_set.address}' is missing a dependency on a `helm_chart` target." + with pytest.raises(ExecutionError, match=msg): + rule_runner.request(HelmChart, [FindHelmDeploymentChart(field_set)]) + + +def test_fail_when_more_than_one_chart_is_found_for_a_deployment(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/foo/BUILD": "helm_chart()", + "src/foo/Chart.yaml": gen_chart_file("foo", version="1.0.0"), + "src/bar/BUILD": "helm_chart()", + "src/bar/Chart.yaml": gen_chart_file("bar", version="1.0.3"), + "src/quxx/BUILD": dedent( + """\ + helm_deployment(dependencies=["//src/foo", "//src/bar"]) + """ + ), + } + ) + + source_root_patterns = ("/src/*",) + rule_runner.set_options([f"--source-root-patterns={repr(source_root_patterns)}"]) + + target = rule_runner.get_target(Address("src/quxx")) + field_set = HelmDeploymentFieldSet.create(target) + + msg = ( + f"The target '{field_set.address}' has too many `{HelmChartTarget.alias}` " + "addresses in its dependencies, it should have only one." + ) + with pytest.raises(ExecutionError, match=msg): + rule_runner.request(HelmChart, [FindHelmDeploymentChart(field_set)]) diff --git a/src/python/pants/backend/helm/util_rules/plugins.py b/src/python/pants/backend/helm/util_rules/plugins.py deleted file mode 100644 index c74c1958a2b6..000000000000 --- a/src/python/pants/backend/helm/util_rules/plugins.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -import logging -from abc import ABCMeta -from dataclasses import dataclass, field -from typing import Any, ClassVar, Generic, Type, TypeVar - -import yaml -from typing_extensions import final - -from pants.backend.helm.util_rules.yaml_utils import snake_case_attr_dict -from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior -from pants.core.util_rules.external_tool import ( - DownloadedExternalTool, - ExternalToolRequest, - TemplatedExternalTool, -) -from pants.engine.collection import Collection -from pants.engine.fs import Digest, DigestContents, DigestSubset, PathGlobs -from pants.engine.internals.selectors import MultiGet -from pants.engine.platform import Platform -from pants.engine.rules import Get, collect_rules, rule -from pants.engine.unions import UnionMembership, union -from pants.option.subsystem import Subsystem -from pants.util.frozendict import FrozenDict -from pants.util.logging import LogLevel -from pants.util.strutil import bullet_list, pluralize - -logger = logging.getLogger(__name__) - - -class HelmPluginMetadataFileNotFound(Exception): - def __init__(self, plugin_name: str) -> None: - super().__init__(f"Helm plugin `{plugin_name}` is missing the `plugin.yaml` metadata file.") - - -class HelmPluginMissingCommand(ValueError): - def __init__(self, plugin_name: str) -> None: - super().__init__( - f"Helm plugin `{plugin_name}` is missing either `platformCommand` entries or a single `command` entry." - ) - - -class HelmPluginSubsystem(Subsystem, metaclass=ABCMeta): - plugin_name: ClassVar[str] - - -class ExternalHelmPlugin(HelmPluginSubsystem, TemplatedExternalTool, metaclass=ABCMeta): - pass - - -@dataclass(frozen=True) -class HelmPluginPlatformCommand: - os: str - arch: str - command: str - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> HelmPluginPlatformCommand: - return cls(**snake_case_attr_dict(d)) - - -@dataclass(frozen=True) -class HelmPluginMetadata: - name: str - version: str - usage: str | None = None - description: str | None = None - ignore_flags: bool | None = None - command: str | None = None - platform_command: tuple[HelmPluginPlatformCommand, ...] = field(default_factory=tuple) - hooks: FrozenDict[str, str] = field(default_factory=FrozenDict) - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> HelmPluginMetadata: - platform_command = [ - HelmPluginPlatformCommand.from_dict(d) for d in d.pop("platformCommand", []) - ] - hooks = d.pop("hooks", {}) - - attrs = snake_case_attr_dict(d) - return cls(platform_command=tuple(platform_command), hooks=FrozenDict(hooks), **attrs) - - @classmethod - def from_bytes(cls, content: bytes) -> HelmPluginMetadata: - return HelmPluginMetadata.from_dict(yaml.safe_load(content)) - - -_ExternalHelmPlugin = TypeVar("_ExternalHelmPlugin", bound=ExternalHelmPlugin) -_GHP = TypeVar("_GHP", bound="ExternalHelmPluginBinding") - - -@union -@dataclass(frozen=True) -class ExternalHelmPluginBinding(Generic[_ExternalHelmPlugin], metaclass=ABCMeta): - plugin_subsystem_cls: ClassVar[Type[ExternalHelmPlugin]] - - name: str - - @final - @classmethod - def create(cls: Type[_GHP]) -> _GHP: - return cls(name=cls.plugin_subsystem_cls.plugin_name) - - -@dataclass(frozen=True) -class ExternalHelmPluginRequest: - plugin_name: str - tool_request: ExternalToolRequest - - @classmethod - def from_subsystem(cls, subsystem: ExternalHelmPlugin) -> ExternalHelmPluginRequest: - return cls( - plugin_name=subsystem.plugin_name, tool_request=subsystem.get_request(Platform.current) - ) - - -@dataclass(frozen=True) -class HelmPlugin: - metadata: HelmPluginMetadata - digest: Digest - - @property - def name(self) -> str: - return self.metadata.name - - @property - def version(self) -> str: - return self.metadata.version - - -class HelmPlugins(Collection[HelmPlugin]): - pass - - -@rule -async def all_helm_plugins(union_membership: UnionMembership) -> HelmPlugins: - bindings = union_membership.get(ExternalHelmPluginBinding) - external_plugins = await MultiGet( - Get(HelmPlugin, ExternalHelmPluginBinding, binding.create()) for binding in bindings - ) - if logger.isEnabledFor(LogLevel.DEBUG.level): - plugins_desc = [f"{p.name}, version: {p.version}" for p in external_plugins] - logger.debug( - f"Downloaded {pluralize(len(external_plugins), 'external Helm plugin')}:\n{bullet_list(plugins_desc)}" - ) - return HelmPlugins(external_plugins) - - -@rule(desc="Download an external Helm plugin", level=LogLevel.DEBUG) -async def download_external_helm_plugin(request: ExternalHelmPluginRequest) -> HelmPlugin: - downloaded_tool = await Get(DownloadedExternalTool, ExternalToolRequest, request.tool_request) - - metadata_file = await Get( - Digest, - DigestSubset( - downloaded_tool.digest, - PathGlobs( - ["plugin.yaml"], - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=f"The Helm plugin `{request.plugin_name}`", - ), - ), - ) - metadata_content = await Get(DigestContents, Digest, metadata_file) - if len(metadata_content) == 0: - raise HelmPluginMetadataFileNotFound(request.plugin_name) - - metadata = HelmPluginMetadata.from_bytes(metadata_content[0].content) - if not metadata.command and not metadata.platform_command: - raise HelmPluginMissingCommand(request.plugin_name) - - return HelmPlugin(metadata=metadata, digest=downloaded_tool.digest) - - -def rules(): - return collect_rules() diff --git a/src/python/pants/backend/helm/util_rules/post_renderer.py b/src/python/pants/backend/helm/util_rules/post_renderer.py new file mode 100644 index 000000000000..f7c94bf8507b --- /dev/null +++ b/src/python/pants/backend/helm/util_rules/post_renderer.py @@ -0,0 +1,134 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from pants.backend.docker.goals.package_image import DockerFieldSet +from pants.backend.docker.subsystems import dockerfile_parser +from pants.backend.docker.subsystems.docker_options import DockerOptions +from pants.backend.docker.util_rules import ( + docker_binary, + docker_build_args, + docker_build_context, + docker_build_env, + dockerfile, +) +from pants.backend.docker.util_rules.docker_build_context import ( + DockerBuildContext, + DockerBuildContextRequest, +) +from pants.backend.helm.dependency_inference.deployment import FirstPartyHelmDeploymentMappings +from pants.backend.helm.subsystems import post_renderer +from pants.backend.helm.subsystems.post_renderer import HelmPostRenderer, SetupHelmPostRenderer +from pants.backend.helm.target_types import HelmDeploymentFieldSet +from pants.engine.addresses import Address, Addresses +from pants.engine.engine_aware import EngineAwareParameter +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import Targets +from pants.util.logging import LogLevel +from pants.util.strutil import bullet_list, softwrap + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class HelmDeploymentPostRendererRequest(EngineAwareParameter): + field_set: HelmDeploymentFieldSet + + def debug_hint(self) -> str | None: + return self.field_set.address.spec + + +@rule(desc="Prepare Helm deployment post-renderer", level=LogLevel.DEBUG) +async def prepare_post_renderer_for_helm_deployment( + request: HelmDeploymentPostRendererRequest, + mappings: FirstPartyHelmDeploymentMappings, + docker_options: DockerOptions, +) -> HelmPostRenderer: + docker_addr_index = mappings.deployment_to_docker_addresses[request.field_set.address] + docker_addresses = [addr for _, addr in docker_addr_index.values()] + + logger.debug( + softwrap( + f""" + Resolving Docker image references for targets: + + {bullet_list([addr.spec for addr in docker_addresses])} + """ + ) + ) + docker_contexts = await MultiGet( + Get( + DockerBuildContext, + DockerBuildContextRequest( + address=addr, + build_upstream_images=False, + ), + ) + for addr in docker_addresses + ) + + docker_targets = await Get(Targets, Addresses(docker_addresses)) + field_sets = [DockerFieldSet.create(tgt) for tgt in docker_targets] + + def resolve_docker_image_ref(address: Address, context: DockerBuildContext) -> str | None: + docker_field_sets = [fs for fs in field_sets if fs.address == address] + if not docker_field_sets: + return None + + docker_field_set = docker_field_sets[0] + image_refs = docker_field_set.image_refs( + default_repository=docker_options.default_repository, + registries=docker_options.registries(), + interpolation_context=context.interpolation_context, + ) + + # Choose first non-latest image reference found, or fallback to 'latest'. + found_ref: str | None = None + fallback_ref: str | None = None + for ref in image_refs: + if ref.endswith(":latest"): + fallback_ref = ref + else: + found_ref = ref + break + + resolved_ref = found_ref or fallback_ref + if resolved_ref: + logger.debug(f"Resolved Docker image ref '{resolved_ref}' for address {address}.") + else: + logger.warning(f"Could not resolve a valid image ref for Docker target {address}.") + + return resolved_ref + + docker_addr_ref_mapping = { + addr: resolve_docker_image_ref(addr, ctx) + for addr, ctx in zip(docker_addresses, docker_contexts) + } + + def find_replacement(value: tuple[str, Address]) -> str | None: + _, addr = value + return docker_addr_ref_mapping.get(addr) + + replacements = docker_addr_index.transform_values(find_replacement) + + return await Get( + HelmPostRenderer, + SetupHelmPostRenderer(replacements, description_of_origin=request.field_set.address.spec), + ) + + +def rules(): + return [ + *collect_rules(), + *docker_binary.rules(), + *docker_build_args.rules(), + *docker_build_context.rules(), + *docker_build_env.rules(), + *dockerfile.rules(), + *dockerfile_parser.rules(), + *post_renderer.rules(), + ] diff --git a/src/python/pants/backend/helm/util_rules/post_renderer_test.py b/src/python/pants/backend/helm/util_rules/post_renderer_test.py new file mode 100644 index 000000000000..17ae261b2e54 --- /dev/null +++ b/src/python/pants/backend/helm/util_rules/post_renderer_test.py @@ -0,0 +1,199 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.docker.target_types import DockerImageTarget +from pants.backend.helm.dependency_inference import deployment as infer_deployment +from pants.backend.helm.subsystems.post_renderer import ( + HELM_POST_RENDERER_CFG_FILENAME, + HelmPostRenderer, +) +from pants.backend.helm.target_types import ( + HelmChartTarget, + HelmDeploymentFieldSet, + HelmDeploymentTarget, +) +from pants.backend.helm.testutil import HELM_CHART_FILE, HELM_TEMPLATE_HELPERS_FILE +from pants.backend.helm.util_rules import post_renderer +from pants.backend.helm.util_rules.post_renderer import HelmDeploymentPostRendererRequest +from pants.backend.helm.util_rules.renderer import ( + HelmDeploymentCmd, + HelmDeploymentRequest, + RenderedHelmFiles, +) +from pants.backend.helm.util_rules.renderer_test import _read_file_from_digest +from pants.backend.helm.util_rules.tool import HelmProcess +from pants.engine.addresses import Address +from pants.engine.process import ProcessResult +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + target_types=[HelmChartTarget, HelmDeploymentTarget, DockerImageTarget], + rules=[ + *infer_deployment.rules(), + *post_renderer.rules(), + QueryRule(HelmPostRenderer, (HelmDeploymentPostRendererRequest,)), + QueryRule(RenderedHelmFiles, (HelmDeploymentRequest,)), + QueryRule(ProcessResult, (HelmProcess,)), + ], + ) + + +def test_can_prepare_post_renderer(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/mychart/BUILD": "helm_chart()", + "src/mychart/Chart.yaml": HELM_CHART_FILE, + "src/mychart/values.yaml": dedent( + """\ + pods: [] + """ + ), + "src/mychart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, + "src/mychart/templates/pod.yaml": dedent( + """\ + {{- $root := . -}} + {{- range $pod := .Values.pods }} + --- + apiVersion: v1 + kind: Pod + metadata: + name: {{ template "fullname" $root }}-{{ $pod.name }} + labels: + chart: "{{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }}" + spec: + initContainers: + - name: myapp-init-container + image: {{ $pod.initContainerImage }} + containers: + - name: busy + image: busybox:1.29 + - name: myapp-container + image: {{ $pod.appImage }} + {{- end }} + """ + ), + "src/deployment/BUILD": "helm_deployment(name='test', dependencies=['//src/mychart'])", + "src/deployment/values.yaml": dedent( + """\ + pods: + - name: foo + initContainerImage: src/image:init_foo + appImage: src/image:app_foo + - name: bar + initContainerImage: src/image:init_bar + appImage: src/image:app_bar + """ + ), + "src/image/BUILD": dedent( + """\ + docker_image(name="init_foo", source="Dockerfile.init") + docker_image(name="app_foo", source="Dockerfile.app") + + docker_image(name="init_bar", source="Dockerfile.init") + docker_image(name="app_bar", source="Dockerfile.app") + """ + ), + "src/image/Dockerfile.init": "FROM busybox:1.28", + "src/image/Dockerfile.app": "FROM busybox:1.28", + } + ) + + source_root_patterns = ("src/*",) + rule_runner.set_options( + [f"--source-root-patterns={repr(source_root_patterns)}"], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + expected_config_file = dedent( + """\ + --- + mychart/templates/pod.yaml: + - paths: + /spec/containers/1/image: app_foo:latest + /spec/initContainers/0/image: init_foo:latest + - paths: + /spec/containers/1/image: app_bar:latest + /spec/initContainers/0/image: init_bar:latest + """ + ) + + expected_rendered_pod = dedent( + """\ + --- + # Source: mychart/templates/pod.yaml + apiVersion: v1 + kind: Pod + metadata: + name: test-mychart-foo + labels: + chart: mychart-0.1.0 + spec: + initContainers: + - name: myapp-init-container + image: init_foo:latest + containers: + - name: busy + image: busybox:1.29 + - name: myapp-container + image: app_foo:latest + --- + # Source: mychart/templates/pod.yaml + apiVersion: v1 + kind: Pod + metadata: + name: test-mychart-bar + labels: + chart: mychart-0.1.0 + spec: + initContainers: + - name: myapp-init-container + image: init_bar:latest + containers: + - name: busy + image: busybox:1.29 + - name: myapp-container + image: app_bar:latest + """ + ) + + deployment_addr = Address("src/deployment", target_name="test") + tgt = rule_runner.get_target(deployment_addr) + field_set = HelmDeploymentFieldSet.create(tgt) + + post_renderer = rule_runner.request( + HelmPostRenderer, + [HelmDeploymentPostRendererRequest(field_set)], + ) + + config_file = _read_file_from_digest( + rule_runner, digest=post_renderer.digest, filename=HELM_POST_RENDERER_CFG_FILENAME + ) + assert config_file == expected_config_file + + rendered_output = rule_runner.request( + RenderedHelmFiles, + [ + HelmDeploymentRequest( + field_set=field_set, + cmd=HelmDeploymentCmd.RENDER, + description="Test post-renderer output", + post_renderer=post_renderer, + ) + ], + ) + assert "mychart/templates/pod.yaml" in rendered_output.snapshot.files + + rendered_pod_file = _read_file_from_digest( + rule_runner, digest=rendered_output.snapshot.digest, filename="mychart/templates/pod.yaml" + ) + assert rendered_pod_file == expected_rendered_pod diff --git a/src/python/pants/backend/helm/util_rules/renderer.py b/src/python/pants/backend/helm/util_rules/renderer.py new file mode 100644 index 000000000000..a6959aaa4988 --- /dev/null +++ b/src/python/pants/backend/helm/util_rules/renderer.py @@ -0,0 +1,406 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import dataclasses +import logging +import os +import re +from collections import defaultdict +from dataclasses import dataclass +from enum import Enum +from itertools import chain +from typing import Any, Iterable, Mapping + +from pants.backend.helm.subsystems import post_renderer +from pants.backend.helm.subsystems.post_renderer import HelmPostRenderer +from pants.backend.helm.target_types import HelmDeploymentFieldSet, HelmDeploymentSourcesField +from pants.backend.helm.util_rules import chart, tool +from pants.backend.helm.util_rules.chart import FindHelmDeploymentChart, HelmChart +from pants.backend.helm.util_rules.tool import HelmProcess +from pants.core.util_rules.source_files import SourceFilesRequest +from pants.core.util_rules.stripped_source_files import StrippedSourceFiles +from pants.engine.addresses import Address +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType +from pants.engine.fs import ( + EMPTY_DIGEST, + EMPTY_SNAPSHOT, + CreateDigest, + Digest, + DigestSubset, + Directory, + FileContent, + MergeDigests, + PathGlobs, + RemovePrefix, + Snapshot, +) +from pants.engine.internals.native_engine import FileDigest +from pants.engine.process import InteractiveProcess, Process, ProcessResult +from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper +from pants.util.logging import LogLevel +from pants.util.meta import frozen_after_init +from pants.util.strutil import pluralize, softwrap + +logger = logging.getLogger(__name__) + + +class HelmDeploymentCmd(Enum): + """Supported Helm rendering commands, for use when creating a `HelmDeploymentRenderer`.""" + + UPGRADE = "upgrade" + RENDER = "template" + + +@dataclass(unsafe_hash=True) +@frozen_after_init +class HelmDeploymentRequest(EngineAwareParameter): + field_set: HelmDeploymentFieldSet + + cmd: HelmDeploymentCmd + description: str = dataclasses.field(compare=False) + extra_argv: tuple[str, ...] + post_renderer: HelmPostRenderer | None + + def __init__( + self, + field_set: HelmDeploymentFieldSet, + *, + cmd: HelmDeploymentCmd, + description: str, + extra_argv: Iterable[str] | None = None, + post_renderer: HelmPostRenderer | None = None, + ) -> None: + self.field_set = field_set + self.cmd = cmd + self.description = description + self.extra_argv = tuple(extra_argv or ()) + self.post_renderer = post_renderer + + def debug_hint(self) -> str | None: + return self.field_set.address.spec + + def metadata(self) -> dict[str, Any] | None: + return { + "cmd": self.cmd.value, + "address": self.field_set.address, + "description": self.description, + "extra_argv": self.extra_argv, + "post_renderer": self.post_renderer, + } + + +@dataclass(frozen=True) +class _HelmDeploymentProcessWrapper(EngineAwareParameter, EngineAwareReturnType): + """Intermediate representation of a `HelmProcess` that will produce a fully rendered set of + manifests from a given chart. + + The encapsulated `process` will be side-effecting dependening on the `cmd` that was originally requested. + + This is meant to only be used internally by this module. + """ + + chart: HelmChart + cmd: HelmDeploymentCmd + process: HelmProcess + address: Address + output_directory: str | None + + @property + def is_side_effect(self) -> bool: + return self.cmd != HelmDeploymentCmd.RENDER + + @property + def uses_post_renderer(self) -> bool: + if self.output_directory: + return False + return True + + def debug_hint(self) -> str | None: + return self.address.spec + + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + + def message(self) -> str | None: + msg = softwrap( + f""" + Built deployment process for {self.address} using chart {self.chart.address} + with{'out' if not self.output_directory else ''} a post-renderer stage + """ + ) + if self.output_directory: + msg += f" and output directory: {self.output_directory}." + else: + msg += " and output to stdout." + return msg + + def metadata(self) -> dict[str, Any] | None: + return { + "address": self.address, + "chart": self.chart, + "process": self.process, + "output_directory": self.output_directory, + } + + +@dataclass(frozen=True) +class RenderedHelmFiles(EngineAwareReturnType): + address: Address + chart: HelmChart + snapshot: Snapshot + + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + + def message(self) -> str | None: + return softwrap( + f""" + Generated {pluralize(len(self.snapshot.files), 'file')} from deployment {self.address} + using chart {self.chart.address}. + """ + ) + + def artifacts(self) -> dict[str, FileDigest | Snapshot] | None: + return {"content": self.snapshot} + + def metadata(self) -> dict[str, Any] | None: + return {"address": self.address, "chart": self.chart} + + +@rule_helper +async def _sort_value_file_names_for_evaluation( + address: Address, *, sources_field: HelmDeploymentSourcesField, value_files_snapshot: Snapshot +) -> list[str]: + """Sorts the list of files in `value_files_snapshot` alphabetically but grouping them in the + order in which they have been given in the `sources_field` field glob patterns.""" + + result: list[str] = [] + + if not sources_field.value: + result = list(value_files_snapshot.files) + result.sort() + else: + # Break the list of filenames in subsets that follow the order given in the `sources` field + subset_snapshots = await MultiGet( + Get(Snapshot, DigestSubset(value_files_snapshot.digest, PathGlobs([glob_pattern]))) + for glob_pattern in sources_field.globs + ) + sources_subsets = [set(snapshot.files) for snapshot in subset_snapshots] + + def minimise_and_sort_subset(input_subset: set[str]) -> list[str]: + result: set[str] = input_subset + for subset in sources_subsets: + if subset == input_subset: + continue + + if result.issuperset(subset): + result = result.difference(subset) + + result_as_list = list(result) + result_as_list.sort() + return result_as_list + + result = list( + chain.from_iterable([minimise_and_sort_subset(subset) for subset in sources_subsets]) + ) + + logger.debug( + softwrap( + f"""Value files for {address} would be evaluated using the following order: + + {', '.join(result)} + """ + ) + ) + + return result + + +@rule(desc="Prepare Helm deployment renderer") +async def setup_render_helm_deployment_process( + request: HelmDeploymentRequest, +) -> _HelmDeploymentProcessWrapper: + chart, value_files = await MultiGet( + Get(HelmChart, FindHelmDeploymentChart(request.field_set)), + Get( + StrippedSourceFiles, + SourceFilesRequest( + sources_fields=[request.field_set.sources], + for_sources_types=[HelmDeploymentSourcesField], + enable_codegen=True, + ), + ), + ) + + logger.debug(f"Using Helm chart {chart.address} in deployment {request.field_set.address}.") + + output_dir = None + output_digest = EMPTY_DIGEST + output_directories = None + if not request.post_renderer: + output_dir = "__out" + output_digest = await Get(Digest, CreateDigest([Directory(output_dir)])) + output_directories = [output_dir] + + # Sort the list of file names following a consistent ordering + sorted_value_files = await _sort_value_file_names_for_evaluation( + request.field_set.address, + sources_field=request.field_set.sources, + value_files_snapshot=value_files.snapshot, + ) + + # Digests to be used as an input into the renderer process. + input_digests = [ + chart.snapshot.digest, + value_files.snapshot.digest, + output_digest, + ] + + # Additional process values in case a post_renderer has been requested. + env: Mapping[str, str] = {} + immutable_input_digests: Mapping[str, Digest] = {} + append_only_caches: Mapping[str, str] = {} + if request.post_renderer: + logger.debug(f"Using post-renderer stage in deployment {request.field_set.address}") + input_digests.append(request.post_renderer.digest) + env = request.post_renderer.env + immutable_input_digests = request.post_renderer.immutable_input_digests + append_only_caches = request.post_renderer.append_only_caches + + merged_digests = await Get(Digest, MergeDigests(input_digests)) + + release_name = request.field_set.release_name.value or request.field_set.address.target_name + inline_values = request.field_set.values.value + + def maybe_escape_string_value(value: str) -> str: + if re.findall("\\s+", value): + return f'"{value}"' + return value + + process = HelmProcess( + argv=[ + request.cmd.value, + release_name, + chart.path, + *( + ("--description", f'"{request.field_set.description.value}"') + if request.field_set.description.value + else () + ), + *( + ("--namespace", request.field_set.namespace.value) + if request.field_set.namespace.value + else () + ), + *(("--create-namespace",) if request.field_set.create_namespace.value else ()), + *(("--skip-crds",) if request.field_set.skip_crds.value else ()), + *(("--no-hooks",) if request.field_set.no_hooks.value else ()), + *(("--output-dir", output_dir) if output_dir else ()), + *( + ("--post-renderer", os.path.join(".", request.post_renderer.exe)) + if request.post_renderer + else () + ), + *(("--values", ",".join(sorted_value_files)) if sorted_value_files else ()), + *chain.from_iterable( + ( + ("--set", f"{key}={maybe_escape_string_value(value)}") + for key, value in inline_values.items() + ) + if inline_values + else () + ), + *request.extra_argv, + ], + extra_env=env, + extra_immutable_input_digests=immutable_input_digests, + extra_append_only_caches=append_only_caches, + description=request.description, + input_digest=merged_digests, + output_directories=output_directories, + ) + + return _HelmDeploymentProcessWrapper( + cmd=request.cmd, + chart=chart, + process=process, + address=request.field_set.address, + output_directory=output_dir, + ) + + +_YAML_FILE_SEPARATOR = "---" +_HELM_OUTPUT_FILE_MARKER = "# Source: " + + +@rule(desc="Render Helm deployment", level=LogLevel.DEBUG) +async def run_renderer(process_wrapper: _HelmDeploymentProcessWrapper) -> RenderedHelmFiles: + assert not process_wrapper.is_side_effect + + def file_content(file_name: str, lines: Iterable[str]) -> FileContent: + sanitised_lines = list(lines) + if len(sanitised_lines) == 0: + return FileContent(file_name, b"") + if sanitised_lines[len(sanitised_lines) - 1] == _YAML_FILE_SEPARATOR: + sanitised_lines = sanitised_lines[:-1] + if sanitised_lines[0] != _YAML_FILE_SEPARATOR: + sanitised_lines = [_YAML_FILE_SEPARATOR, *sanitised_lines] + + content = "\n".join(sanitised_lines) + "\n" + return FileContent(file_name, content.encode("utf-8")) + + def parse_renderer_output(result: ProcessResult) -> list[FileContent]: + rendered_files_contents = result.stdout.decode("utf-8") + rendered_files: dict[str, list[str]] = defaultdict(list) + + curr_file_name = None + for line in rendered_files_contents.splitlines(): + if not line: + continue + + if line.startswith(_HELM_OUTPUT_FILE_MARKER): + curr_file_name = line[len(_HELM_OUTPUT_FILE_MARKER) :] + + if not curr_file_name: + continue + + rendered_files[curr_file_name].append(line) + + return [file_content(file_name, lines) for file_name, lines in rendered_files.items()] + + logger.debug(f"Rendering Helm files for {process_wrapper.address}") + result = await Get(ProcessResult, HelmProcess, process_wrapper.process) + + output_snapshot = EMPTY_SNAPSHOT + if not process_wrapper.output_directory: + logger.debug( + f"Parsing Helm rendered files from the process' output of {process_wrapper.address}." + ) + output_snapshot = await Get(Snapshot, CreateDigest(parse_renderer_output(result))) + else: + logger.debug( + f"Obtaining Helm rendered files from the process' output directory of {process_wrapper.address}." + ) + output_snapshot = await Get( + Snapshot, RemovePrefix(result.output_digest, process_wrapper.output_directory) + ) + + return RenderedHelmFiles( + address=process_wrapper.address, chart=process_wrapper.chart, snapshot=output_snapshot + ) + + +@rule +async def materialize_deployment_process_wrapper_into_interactive_process( + process_wrapper: _HelmDeploymentProcessWrapper, +) -> InteractiveProcess: + assert process_wrapper.is_side_effect + + process = await Get(Process, HelmProcess, process_wrapper.process) + return InteractiveProcess.from_process(process) + + +def rules(): + return [*collect_rules(), *chart.rules(), *tool.rules(), *post_renderer.rules()] diff --git a/src/python/pants/backend/helm/util_rules/renderer_test.py b/src/python/pants/backend/helm/util_rules/renderer_test.py new file mode 100644 index 000000000000..c20cdeeca32d --- /dev/null +++ b/src/python/pants/backend/helm/util_rules/renderer_test.py @@ -0,0 +1,194 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from pathlib import PurePath +from textwrap import dedent + +import pytest + +from pants.backend.helm.target_types import ( + HelmChartTarget, + HelmDeploymentFieldSet, + HelmDeploymentTarget, +) +from pants.backend.helm.testutil import HELM_CHART_FILE, HELM_TEMPLATE_HELPERS_FILE +from pants.backend.helm.util_rules import renderer +from pants.backend.helm.util_rules.renderer import ( + HelmDeploymentCmd, + HelmDeploymentRequest, + RenderedHelmFiles, +) +from pants.core.util_rules import external_tool, stripped_source_files +from pants.engine.addresses import Address +from pants.engine.fs import DigestContents, DigestSubset, PathGlobs +from pants.engine.internals.native_engine import Digest +from pants.engine.process import InteractiveProcess +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + target_types=[HelmChartTarget, HelmDeploymentTarget], + rules=[ + *external_tool.rules(), + *stripped_source_files.rules(), + *renderer.rules(), + QueryRule(InteractiveProcess, (HelmDeploymentRequest,)), + QueryRule(RenderedHelmFiles, (HelmDeploymentRequest,)), + ], + ) + source_root_patterns = ("src/*",) + rule_runner.set_options( + [f"--source-root-patterns={repr(source_root_patterns)}"], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + return rule_runner + + +def _read_file_from_digest(rule_runner: RuleRunner, *, digest: Digest, filename: str) -> str: + config_file_digest = rule_runner.request(Digest, [DigestSubset(digest, PathGlobs([filename]))]) + config_file_contents = rule_runner.request(DigestContents, [config_file_digest]) + return config_file_contents[0].content.decode("utf-8") + + +def _common_workspace_files() -> dict[str | PurePath, str | bytes]: + return { + "src/mychart/BUILD": "helm_chart()", + "src/mychart/Chart.yaml": HELM_CHART_FILE, + "src/mychart/values.yaml": dedent( + """\ + config_maps: + - name: foo + data: + foo_key: foo_value + - name: bar + data: + bar_key: bar_value + """ + ), + "src/mychart/templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, + "src/mychart/templates/configmap.yaml": dedent( + """\ + {{- $root := . -}} + {{- $allConfigMaps := .Values.config_maps -}} + {{- range $configMap := $allConfigMaps }} + --- + {{- with $configMap }} + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ template "fullname" $root }}-{{ .name }} + labels: + chart: "{{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }}" + data: + {{ toYaml .data | indent 2 }} + {{- end }} + {{- end }} + """ + ), + } + + +def test_sort_deployment_files_alphabetically(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + **_common_workspace_files(), + "src/deployment/BUILD": "helm_deployment(name='foo', dependencies=['//src/mychart'])", + "src/deployment/b.yaml": "", + "src/deployment/a.yaml": "", + } + ) + + tgt = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(tgt) + + render_request = HelmDeploymentRequest( + cmd=HelmDeploymentCmd.UPGRADE, + field_set=field_set, + description="Test sort files using default sources", + ) + + render_process = rule_runner.request(InteractiveProcess, [render_request]) + assert "a.yaml,b.yaml" in render_process.process.argv + + +def test_sort_deployment_files_as_given(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + **_common_workspace_files(), + "src/deployment/BUILD": "helm_deployment(name='foo', dependencies=['//src/mychart'], sources=['b.yaml', '*.yaml'])", + "src/deployment/b.yaml": "", + "src/deployment/a.yaml": "", + } + ) + + tgt = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(tgt) + + render_request = HelmDeploymentRequest( + cmd=HelmDeploymentCmd.UPGRADE, + field_set=field_set, + description="Test sort files using default sources", + ) + + render_process = rule_runner.request(InteractiveProcess, [render_request]) + assert "b.yaml,a.yaml" in render_process.process.argv + + +def test_renders_files(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + **_common_workspace_files(), + "src/deployment/BUILD": "helm_deployment(name='foo', dependencies=['//src/mychart'])", + } + ) + + tgt = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(tgt) + + expected_config_map_file = dedent( + """\ + --- + # Source: mychart/templates/configmap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: foo-mychart-foo + labels: + chart: "mychart-0.1.0" + data: + foo_key: foo_value + --- + # Source: mychart/templates/configmap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: foo-mychart-bar + labels: + chart: "mychart-0.1.0" + data: + bar_key: bar_value + """ + ) + + render_request = HelmDeploymentRequest( + cmd=HelmDeploymentCmd.RENDER, + field_set=field_set, + description="Test template rendering", + ) + + rendered = rule_runner.request(RenderedHelmFiles, [render_request]) + + assert rendered.snapshot.files + assert "mychart/templates/configmap.yaml" in rendered.snapshot.files + + template_output = _read_file_from_digest( + rule_runner, + digest=rendered.snapshot.digest, + filename="mychart/templates/configmap.yaml", + ) + assert template_output == expected_config_map_file diff --git a/src/python/pants/backend/helm/util_rules/sources.py b/src/python/pants/backend/helm/util_rules/sources.py index e8e6a4dd0848..bd33c19848e3 100644 --- a/src/python/pants/backend/helm/util_rules/sources.py +++ b/src/python/pants/backend/helm/util_rules/sources.py @@ -11,6 +11,7 @@ HelmChartSourcesField, ) from pants.core.target_types import FileSourceField, ResourceSourceField +from pants.core.util_rules import stripped_source_files from pants.core.util_rules.source_files import SourceFilesRequest from pants.core.util_rules.stripped_source_files import StrippedSourceFiles from pants.engine.fs import MergeDigests, Snapshot @@ -112,4 +113,4 @@ async def get_helm_source_files(request: HelmChartSourceFilesRequest) -> HelmCha def rules(): - return collect_rules() + return [*collect_rules(), *stripped_source_files.rules()] diff --git a/src/python/pants/backend/helm/util_rules/sources_test.py b/src/python/pants/backend/helm/util_rules/sources_test.py index 019d8785c975..40076abb6d54 100644 --- a/src/python/pants/backend/helm/util_rules/sources_test.py +++ b/src/python/pants/backend/helm/util_rules/sources_test.py @@ -14,13 +14,12 @@ HELM_TEMPLATE_HELPERS_FILE, HELM_VALUES_FILE, K8S_CRD_FILE, - K8S_SERVICE_FILE, + K8S_SERVICE_TEMPLATE, ) from pants.backend.helm.util_rules import sources from pants.backend.helm.util_rules.sources import HelmChartSourceFiles, HelmChartSourceFilesRequest from pants.build_graph.address import Address from pants.core.target_types import FilesGeneratorTarget, ResourcesGeneratorTarget -from pants.core.util_rules import stripped_source_files from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -31,7 +30,6 @@ def rule_runner() -> RuleRunner: target_types=[HelmChartTarget, ResourcesGeneratorTarget, FilesGeneratorTarget], rules=[ *sources.rules(), - *stripped_source_files.rules(), QueryRule(HelmChartSourceFiles, (HelmChartSourceFilesRequest,)), ], ) @@ -51,7 +49,7 @@ def test_source_templates_are_always_included(rule_runner: RuleRunner) -> None: "values.yaml": HELM_VALUES_FILE, "crds/foo.yml": K8S_CRD_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, "resource.xml": "", "file.txt": "", } @@ -97,7 +95,7 @@ def test_source_templates_includes( "Chart.yaml": HELM_CHART_FILE, "values.yaml": HELM_VALUES_FILE, "templates/_helpers.tpl": HELM_TEMPLATE_HELPERS_FILE, - "templates/service.yaml": K8S_SERVICE_FILE, + "templates/service.yaml": K8S_SERVICE_TEMPLATE, "resource.xml": "", "file.txt": "", } diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index ff9304a6e7c6..7c88dbbd577d 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -4,29 +4,281 @@ from __future__ import annotations import dataclasses +import logging import os +from abc import ABCMeta from dataclasses import dataclass -from typing import Iterable, Mapping +from typing import Any, ClassVar, Generic, Iterable, Mapping, Type, TypeVar + +import yaml +from typing_extensions import final from pants.backend.helm.subsystems.helm import HelmSubsystem -from pants.backend.helm.util_rules.plugins import HelmPlugins -from pants.backend.helm.util_rules.plugins import rules as plugins_rules -from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest +from pants.backend.helm.utils.yaml import snake_case_attr_dict +from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior +from pants.core.util_rules import external_tool +from pants.core.util_rules.external_tool import ( + DownloadedExternalTool, + ExternalToolRequest, + TemplatedExternalTool, +) +from pants.engine import process +from pants.engine.collection import Collection +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.environment import Environment, EnvironmentRequest -from pants.engine.fs import CreateDigest, Digest, DigestSubset, Directory, PathGlobs, RemovePrefix -from pants.engine.internals.native_engine import AddPrefix, MergeDigests +from pants.engine.fs import ( + CreateDigest, + Digest, + DigestContents, + DigestSubset, + Directory, + FileDigest, + PathGlobs, + RemovePrefix, +) +from pants.engine.internals.native_engine import AddPrefix, MergeDigests, Snapshot from pants.engine.platform import Platform from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionMembership, union +from pants.option.subsystem import Subsystem from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.meta import frozen_after_init +from pants.util.strutil import bullet_list, pluralize + +logger = logging.getLogger(__name__) _HELM_CACHE_NAME = "helm" _HELM_CACHE_DIR = "__cache" _HELM_CONFIG_DIR = "__config" _HELM_DATA_DIR = "__data" +# --------------------------------------------- +# Helm Plugins Support +# --------------------------------------------- + + +class HelmPluginMetadataFileNotFound(Exception): + def __init__(self, plugin_name: str) -> None: + super().__init__(f"Helm plugin `{plugin_name}` is missing the `plugin.yaml` metadata file.") + + +class HelmPluginMissingCommand(ValueError): + def __init__(self, plugin_name: str) -> None: + super().__init__( + f"Helm plugin `{plugin_name}` is missing either `platformCommand` entries or a single `command` entry." + ) + + +class HelmPluginSubsystem(Subsystem, metaclass=ABCMeta): + """Base class for any kind of Helm plugin.""" + + plugin_name: ClassVar[str] + + +class ExternalHelmPlugin(HelmPluginSubsystem, TemplatedExternalTool, metaclass=ABCMeta): + """Represents the subsystem for a Helm plugin that needs to be downloaded from an external + source. + + For declaring an External Helm plugin, extend this class provinding a value of the + `plugin_name` class attribute and implement the rest of it like you would do for + any other `TemplatedExternalTool`. + + This class is meant to be used in combination with `ExternalHelmPluginBinding`, as + in the following example: + + class MyHelmPluginSubsystem(ExternalHelmPlugin): + plugin_name = "myplugin" + options_scope = "my_plugin" + help = "..." + + ... + + + class MyPluginBinding(ExternalHelmPluginBinding[MyPluginSubsystem]): + plugin_subsystem_cls = MyHelmPluginSubsystem + + With that class structure, then define a `UnionRule` so Pants can find this plugin and + use it in the Helm setup: + + @rule + def download_myplugin_plugin_request( + _: MyPluginBinding, subsystem: MyHelmPluginSubsystem + ) -> ExternalHelmPluginRequest: + return ExternalHelmPluginRequest.from_subsystem(subsystem) + + + def rules(): + return [ + *collect_rules(), + UnionRule(ExternalHelmPluginBinding, MyPluginBinding), + ] + """ + + +@dataclass(frozen=True) +class HelmPluginPlatformCommand: + os: str + arch: str + command: str + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> HelmPluginPlatformCommand: + return cls(**snake_case_attr_dict(d)) + + +@dataclass(frozen=True) +class HelmPluginInfo: + name: str + version: str + usage: str | None = None + description: str | None = None + ignore_flags: bool | None = None + command: str | None = None + platform_command: tuple[HelmPluginPlatformCommand, ...] = dataclasses.field( + default_factory=tuple + ) + hooks: FrozenDict[str, str] = dataclasses.field(default_factory=FrozenDict) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> HelmPluginInfo: + platform_command = [ + HelmPluginPlatformCommand.from_dict(d) for d in d.pop("platformCommand", []) + ] + hooks = d.pop("hooks", {}) + + attrs = snake_case_attr_dict(d) + return cls(platform_command=tuple(platform_command), hooks=FrozenDict(hooks), **attrs) + + @classmethod + def from_bytes(cls, content: bytes) -> HelmPluginInfo: + return HelmPluginInfo.from_dict(yaml.safe_load(content)) + + +_ExternalHelmPlugin = TypeVar("_ExternalHelmPlugin", bound=ExternalHelmPlugin) +_EHPB = TypeVar("_EHPB", bound="ExternalHelmPluginBinding") + + +@union +@dataclass(frozen=True) +class ExternalHelmPluginBinding(Generic[_ExternalHelmPlugin], metaclass=ABCMeta): + """Union type allowing Pants to discover global external Helm plugins.""" + + plugin_subsystem_cls: ClassVar[Type[ExternalHelmPlugin]] + + name: str + + @final + @classmethod + def create(cls: Type[_EHPB]) -> _EHPB: + return cls(name=cls.plugin_subsystem_cls.plugin_name) + + +@dataclass(frozen=True) +class ExternalHelmPluginRequest(EngineAwareParameter): + """Helper class to create a download request for an external Helm plugin.""" + + plugin_name: str + platform: Platform + + _tool_request: ExternalToolRequest + + @classmethod + def from_subsystem(cls, subsystem: ExternalHelmPlugin) -> ExternalHelmPluginRequest: + platform = Platform.current + return cls( + plugin_name=subsystem.plugin_name, + platform=platform, + _tool_request=subsystem.get_request(platform), + ) + + def debug_hint(self) -> str | None: + return self.plugin_name + + def metadata(self) -> dict[str, Any] | None: + return {"platform": self.platform, "url": self._tool_request.download_file_request.url} + + +@dataclass(frozen=True) +class HelmPlugin(EngineAwareReturnType): + info: HelmPluginInfo + platform: Platform + snapshot: Snapshot + + @property + def name(self) -> str: + return self.info.name + + @property + def version(self) -> str: + return self.info.version + + def level(self) -> LogLevel | None: + return LogLevel.INFO + + def message(self) -> str | None: + return f"Materialized Helm plugin {self.name} with version {self.version} for {self.platform} platform." + + def metadata(self) -> dict[str, Any] | None: + return {"name": self.name, "version": self.version, "platform": self.platform} + + def artifacts(self) -> dict[str, FileDigest | Snapshot] | None: + return {"content": self.snapshot} + + def cacheable(self) -> bool: + return True + + +class HelmPlugins(Collection[HelmPlugin]): + pass + + +@rule +async def all_helm_plugins(union_membership: UnionMembership) -> HelmPlugins: + bindings = union_membership.get(ExternalHelmPluginBinding) + external_plugins = await MultiGet( + Get(HelmPlugin, ExternalHelmPluginBinding, binding.create()) for binding in bindings + ) + if logger.isEnabledFor(LogLevel.DEBUG.level): + plugins_desc = [f"{p.name}, version: {p.version}" for p in external_plugins] + logger.debug( + f"Downloaded {pluralize(len(external_plugins), 'external Helm plugin')}:\n{bullet_list(plugins_desc)}" + ) + return HelmPlugins(external_plugins) + + +@rule(desc="Download external Helm plugin", level=LogLevel.DEBUG) +async def download_external_helm_plugin(request: ExternalHelmPluginRequest) -> HelmPlugin: + downloaded_tool = await Get(DownloadedExternalTool, ExternalToolRequest, request._tool_request) + + plugin_info_file = await Get( + Digest, + DigestSubset( + downloaded_tool.digest, + PathGlobs( + ["plugin.yaml", "plugin.yml"], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=request.plugin_name, + ), + ), + ) + plugin_info_contents = await Get(DigestContents, Digest, plugin_info_file) + if len(plugin_info_contents) == 0: + raise HelmPluginMetadataFileNotFound(request.plugin_name) + + plugin_info = HelmPluginInfo.from_bytes(plugin_info_contents[0].content) + if not plugin_info.command and not plugin_info.platform_command: + raise HelmPluginMissingCommand(request.plugin_name) + + plugin_snapshot = await Get(Snapshot, Digest, downloaded_tool.digest) + return HelmPlugin(info=plugin_info, platform=request.platform, snapshot=plugin_snapshot) + + +# --------------------------------------------- +# Helm Binary setup +# --------------------------------------------- + @frozen_after_init @dataclass(unsafe_hash=True) @@ -70,6 +322,7 @@ class HelmProcess: level: LogLevel extra_env: FrozenDict[str, str] extra_immutable_input_digests: FrozenDict[str, Digest] + extra_append_only_caches: FrozenDict[str, str] cache_scope: ProcessCacheScope | None timeout_seconds: int | None output_directories: tuple[str, ...] @@ -86,6 +339,7 @@ def __init__( output_files: Iterable[str] | None = None, extra_env: Mapping[str, str] | None = None, extra_immutable_input_digests: Mapping[str, Digest] | None = None, + extra_append_only_caches: Mapping[str, str] | None = None, cache_scope: ProcessCacheScope | None = None, timeout_seconds: int | None = None, ): @@ -97,6 +351,7 @@ def __init__( self.output_files = tuple(output_files or ()) self.extra_env = FrozenDict(extra_env or {}) self.extra_immutable_input_digests = FrozenDict(extra_immutable_input_digests or {}) + self.extra_append_only_caches = FrozenDict(extra_append_only_caches or {}) self.cache_scope = cache_scope self.timeout_seconds = timeout_seconds @@ -133,10 +388,13 @@ async def setup_helm(helm_subsytem: HelmSubsystem, global_plugins: HelmPlugins) # Install all global Helm plugins if global_plugins: + logger.debug(f"Installing {pluralize(len(global_plugins), 'global Helm plugin')}.") prefixed_plugins_digests = await MultiGet( Get( Digest, - AddPrefix(plugin.digest, os.path.join(_HELM_DATA_DIR, "plugins", plugin.name)), + AddPrefix( + plugin.snapshot.digest, os.path.join(_HELM_DATA_DIR, "plugins", plugin.name) + ), ) for plugin in global_plugins ) @@ -183,6 +441,8 @@ def helm_process(request: HelmProcess, helm_binary: HelmBinary) -> Process: **request.extra_immutable_input_digests, } + append_only_caches = {**helm_binary.append_only_caches, **request.extra_append_only_caches} + return Process( [helm_binary.path, *request.argv], input_digest=request.input_digest, @@ -190,7 +450,7 @@ def helm_process(request: HelmProcess, helm_binary: HelmBinary) -> Process: env=env, description=request.description, level=request.level, - append_only_caches=helm_binary.append_only_caches, + append_only_caches=append_only_caches, output_directories=request.output_directories, output_files=request.output_files, cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL, @@ -199,4 +459,4 @@ def helm_process(request: HelmProcess, helm_binary: HelmBinary) -> Process: def rules(): - return [*collect_rules(), *plugins_rules()] + return [*collect_rules(), *external_tool.rules(), *process.rules()] diff --git a/src/python/pants/backend/helm/util_rules/tool_test.py b/src/python/pants/backend/helm/util_rules/tool_test.py index 40fd89fdba51..422a71be5fab 100644 --- a/src/python/pants/backend/helm/util_rules/tool_test.py +++ b/src/python/pants/backend/helm/util_rules/tool_test.py @@ -8,11 +8,9 @@ from pants.backend.helm.subsystems.helm import HelmSubsystem from pants.backend.helm.util_rules import tool from pants.backend.helm.util_rules.tool import HelmBinary, HelmProcess -from pants.core.util_rules import config_files, external_tool -from pants.engine import process -from pants.engine.internals.native_engine import EMPTY_DIGEST +from pants.engine.fs import EMPTY_DIGEST from pants.engine.platform import Platform -from pants.engine.process import Process, ProcessCacheScope, ProcessResult +from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner from pants.util.frozendict import FrozenDict @@ -23,14 +21,10 @@ def rule_runner() -> RuleRunner: return RuleRunner( target_types=[], rules=[ - *config_files.rules(), - *external_tool.rules(), *tool.rules(), - *process.rules(), QueryRule(HelmBinary, ()), QueryRule(HelmSubsystem, ()), QueryRule(Process, (HelmProcess,)), - QueryRule(ProcessResult, (HelmProcess,)), ], ) diff --git a/src/python/pants/backend/helm/util_rules/yaml_utils.py b/src/python/pants/backend/helm/util_rules/yaml_utils.py deleted file mode 100644 index e7bf6051f581..000000000000 --- a/src/python/pants/backend/helm/util_rules/yaml_utils.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from typing import Any, Mapping - - -def _to_snake_case(str: str) -> str: - """Translates an camel-case or kebab-case identifier by a snake-case one.""" - base_string = str.replace("-", "_") - - result = "" - idx = 0 - for c in base_string: - char_to_add = c - if char_to_add.isupper(): - char_to_add = c.lower() - if idx > 0: - result += "_" - result += char_to_add - idx += 1 - - return result - - -def snake_case_attr_dict(d: Mapping[str, Any]) -> dict[str, Any]: - """Transforms all keys in the given mapping to be snake-case.""" - return {_to_snake_case(name): value for name, value in d.items()} diff --git a/src/python/pants/backend/helm/utils/BUILD b/src/python/pants/backend/helm/utils/BUILD new file mode 100644 index 000000000000..f9b4a7be9f0f --- /dev/null +++ b/src/python/pants/backend/helm/utils/BUILD @@ -0,0 +1,6 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() + +python_tests(name="tests") \ No newline at end of file diff --git a/src/python/pants/backend/helm/utils/__init__.py b/src/python/pants/backend/helm/utils/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/python/pants/backend/helm/utils/yaml.py b/src/python/pants/backend/helm/utils/yaml.py new file mode 100644 index 000000000000..54ce663f094f --- /dev/null +++ b/src/python/pants/backend/helm/utils/yaml.py @@ -0,0 +1,273 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from abc import ABCMeta +from collections import defaultdict +from dataclasses import dataclass +from pathlib import PurePath +from typing import Any, Callable, Generic, Iterable, Iterator, Mapping, Optional, Type, TypeVar + +from pants.engine.collection import Collection +from pants.util.frozendict import FrozenDict +from pants.util.meta import frozen_after_init + + +@dataclass(unsafe_hash=True) +@frozen_after_init +class YamlPath: + """Simple implementation of YAML paths using `/` syntax and being the single slash the path to + the root.""" + + _elements: tuple[str, ...] + _absolute: bool + + def __init__(self, elements: Iterable[str], *, absolute: bool) -> None: + self._elements = tuple(elements) + self._absolute = absolute + + if len(self._elements) == 0 and not self._absolute: + raise ValueError("Relative YAML paths with no elements are not allowed.") + + @classmethod + def parse(cls, path: str) -> YamlPath: + """Parses a YAML path.""" + + is_absolute = path.startswith("/") + return cls([elem for elem in path.split("/") if elem], absolute=is_absolute) + + @classmethod + def root(cls) -> YamlPath: + """Returns a YamlPath that represents the root element.""" + + return cls([], absolute=True) + + @classmethod + def index(cls, idx: int) -> YamlPath: + """Returns a relative YamlPath for the index value provided.""" + + return cls([str(idx)], absolute=False) + + @property + def parent(self) -> YamlPath | None: + """Returns the path to the parent element unless this path is already the root.""" + + if not self.is_root: + return YamlPath(self._elements[:-1], absolute=self._absolute) + return None + + @property + def current(self) -> str: + """Returns the name of the current element referenced by this path. + + The root element will return the empty string. + """ + + if self.is_root: + return "" + return self._elements[len(self._elements) - 1] + + @property + def is_absolute(self) -> bool: + """Returns `True` if this is an absolute path.""" + + return self._absolute + + @property + def is_root(self) -> bool: + """Returns `True` if this path represents the root element.""" + + return len(self._elements) == 0 and self._absolute + + @property + def is_index(self) -> bool: + """Returns `True` if this path is referencing an indexed item inside an array.""" + + try: + int(self.current) + return True + except ValueError: + return False + + def to_relative(self) -> YamlPath: + """Transforms this YamlPath instance into a relative path.""" + + if not self._absolute: + return self + return YamlPath(self._elements, absolute=False) + + def __truediv__(self, other: str | int | YamlPath) -> YamlPath: + if isinstance(other, str): + other_path = YamlPath.parse(other) + elif isinstance(other, int): + other_path = YamlPath.index(other) + else: + other_path = other + + if other_path._absolute: + raise ValueError("Can not append an absolute path to another path.") + + return YamlPath(self._elements + other_path._elements, absolute=self._absolute) + + def __iter__(self): + return iter(self._elements) + + def __str__(self) -> str: + path = "/".join(self._elements) + if self._absolute: + path = f"/{path}" + return path + + +@dataclass(frozen=True) +class YamlElement(metaclass=ABCMeta): + """Abstract base class for elements read from YAML files. + + `element_path` represents the location inside the YAML file where this element is. + """ + + element_path: YamlPath + + +T = TypeVar("T") +R = TypeVar("R") + + +class MutableYamlIndex(Generic[T]): + """Represents a mutable collection of items that is indexed by the following keys: + + - the relative path of the YAML file + - the document index inside the YAML file + - the YAML path of the item + """ + + _data: dict[PurePath, dict[int, dict[YamlPath, T]]] + + def __init__(self) -> None: + self._data = defaultdict(dict) + + def insert( + self, *, file_path: PurePath, yaml_path: YamlPath, item: T, document_index: int = 0 + ) -> None: + """Inserts an item at the given position in the index.""" + + doc_index = self._data[file_path].get(document_index, {}) + if not doc_index: + self._data[file_path][document_index] = doc_index + + doc_index[yaml_path] = item + + def frozen(self) -> FrozenYamlIndex[T]: + """Transforms this collection into a frozen (immutable) one.""" + + return FrozenYamlIndex(self) + + +@dataclass(frozen=True) +class _YamlDocumentIndexNode(Generic[T]): + """Helper node item for the `FrozenYamlIndex` type.""" + + paths: FrozenDict[YamlPath, T] + + @classmethod + def empty(cls: Type[_YamlDocumentIndexNode[T]]) -> _YamlDocumentIndexNode[T]: + return cls(paths=FrozenDict()) + + def to_json_dict(self) -> dict[str, dict[str, str]]: + items_dict: dict[str, str] = {} + for path, item in self.paths.items(): + items_dict[str(path)] = str(item) + return {"paths": items_dict} + + +@frozen_after_init +class FrozenYamlIndex(Generic[T]): + """Represents a frozen collection of items that is indexed by the following keys: + + - the relative path of the YAML file + - the document index inside the YAML file + - the YAML path of the item + """ + + _data: FrozenDict[PurePath, Collection[_YamlDocumentIndexNode[T]]] + + def __init__(self, other: MutableYamlIndex[T]) -> None: + data: dict[PurePath, Collection[_YamlDocumentIndexNode[T]]] = {} + for file_path, doc_index in other._data.items(): + max_index = max(doc_index.keys()) + doc_list: list[_YamlDocumentIndexNode[T]] = [_YamlDocumentIndexNode.empty()] * ( + max_index + 1 + ) + + for idx, item_map in doc_index.items(): + doc_list[idx] = _YamlDocumentIndexNode(paths=FrozenDict(item_map)) + + data[file_path] = Collection(doc_list) + self._data = FrozenDict(data) + + def _items(self) -> Iterator[tuple[PurePath, int, YamlPath, T]]: + for file_path, doc_indexes in self._data.items(): + for idx, doc_index in enumerate(doc_indexes): + for yaml_path, item in doc_index.paths.items(): + yield file_path, idx, yaml_path, item + + def transform_values(self, func: Callable[[T], Optional[R]]) -> FrozenYamlIndex[R]: + """Transforms the values of the given indexed collection into those that are returned from + the received function. + + The items that map to `None` in the given function are not included in the result. + + This is a combination of the `map` and `filter` higher-order functions into one so + both operations are performed in a single pass. + """ + + mutable_index: MutableYamlIndex[R] = MutableYamlIndex() + for file_path, doc_index, yaml_path, item in self._items(): + new_item = func(item) + if new_item is not None: + mutable_index.insert( + file_path=file_path, + document_index=doc_index, + yaml_path=yaml_path, + item=new_item, + ) + return mutable_index.frozen() + + def values(self) -> Iterator[T]: + """Returns an iterator over the values of this index.""" + + for _, _, _, item in self._items(): + yield item + + def to_json_dict(self) -> dict[str, Any]: + """Transforms this collection into a JSON-like dictionary that can be dumped later.""" + + result = {} + for file_path, documents in self._data.items(): + result[str(file_path)] = [doc_idx.to_json_dict() for doc_idx in documents] + return result + + +def _to_snake_case(str: str) -> str: + """Translates a camel-case or kebab-case identifier into a snake-case one.""" + + base_string = str.replace("-", "_") + + result = "" + idx = 0 + for c in base_string: + char_to_add = c + if char_to_add.isupper(): + char_to_add = c.lower() + if idx > 0: + result += "_" + result += char_to_add + idx += 1 + + return result + + +def snake_case_attr_dict(d: Mapping[str, Any]) -> dict[str, Any]: + """Transforms all keys in the given mapping to be snake-case.""" + return {_to_snake_case(name): value for name, value in d.items()} diff --git a/src/python/pants/backend/helm/utils/yaml_test.py b/src/python/pants/backend/helm/utils/yaml_test.py new file mode 100644 index 000000000000..5c47d299acb2 --- /dev/null +++ b/src/python/pants/backend/helm/utils/yaml_test.py @@ -0,0 +1,46 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import pytest + +from pants.backend.helm.utils.yaml import YamlPath + + +@pytest.mark.parametrize( + "path, is_absolute, is_root, is_index", + [ + ("/", True, True, False), + ("/0", True, False, True), + ("/path", True, False, False), + ("0", False, False, True), + ("path", False, False, False), + ], +) +def test_yaml_path_parser(path: str, is_absolute: bool, is_root: bool, is_index: bool) -> None: + parsed = YamlPath.parse(path) + + assert parsed.is_absolute == is_absolute + assert parsed.is_root == is_root + assert parsed.is_index == is_index + + if parsed.is_root: + assert not parsed.parent + + nested = parsed / "nested" + + assert not nested.is_root + assert nested.parent == parsed + + if not parsed.is_absolute: + absolute = YamlPath.root() / parsed + + assert absolute.is_absolute + + if parsed.is_index: + index = YamlPath.index(int(parsed.current)) + + assert not index.is_absolute + assert index.is_index + assert index.current == parsed.current diff --git a/src/python/pants/core/util_rules/system_binaries.py b/src/python/pants/core/util_rules/system_binaries.py index f5958a2ce400..c1fa45e9631c 100644 --- a/src/python/pants/core/util_rules/system_binaries.py +++ b/src/python/pants/core/util_rules/system_binaries.py @@ -334,6 +334,10 @@ class MvBinary(BinaryPath): pass +class CatBinary(BinaryPath): + pass + + class ChmodBinary(BinaryPath): pass @@ -687,6 +691,14 @@ async def find_tar() -> TarBinary: return TarBinary(first_path.path, first_path.fingerprint) +@rule(desc="Finding the `cat` binary", level=LogLevel.DEBUG) +async def find_cat() -> CatBinary: + request = BinaryPathRequest(binary_name="cat", search_path=SEARCH_PATHS) + paths = await Get(BinaryPaths, BinaryPathRequest, request) + first_path = paths.first_path_or_raise(request, rationale="outputing content from files") + return CatBinary(first_path.path, first_path.fingerprint) + + @rule(desc="Finding the `mkdir` binary", level=LogLevel.DEBUG) async def find_mkdir() -> MkdirBinary: request = BinaryPathRequest(binary_name="mkdir", search_path=SEARCH_PATHS) @@ -776,7 +788,10 @@ class GunzipBinaryRequest: pass -@dataclass(frozen=True) +class CatBinaryRequest: + pass + + class TarBinaryRequest: pass @@ -826,6 +841,11 @@ async def find_tar_wrapper(_: TarBinaryRequest, tar_binary: TarBinary) -> TarBin return tar_binary +@rule +async def find_cat_wrapper(_: CatBinaryRequest, cat_binary: CatBinary) -> CatBinary: + return cat_binary + + @rule async def find_mkdir_wrapper(_: MkdirBinaryRequest, mkdir_binary: MkdirBinary) -> MkdirBinary: return mkdir_binary