From ed4dffcd70a4a6626a39507df0665a9a82ad2155 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 20 Jun 2022 15:38:13 +0100 Subject: [PATCH] [FEATURE] - Exposing Integrations (#180) * [FEATURE] - Exposing Integrations At present the reports by infracost and checkov are only available in the controller namespace which make reporting on them difficult, beyond the pod logs. In the intention is to have a a tnctl command (a kubectl plugin) and allow uses to explain / describe the configuration resource --- .gitignore | 1 + pkg/controller/configuration/ensure.go | 101 +++++++++++++----- .../configuration/reconcile_test.go | 35 ++++++ test/e2e/check-suite.sh | 1 - test/e2e/integration/checkov.bats | 9 ++ test/e2e/integration/cloud/aws/infracost.bats | 5 +- test/e2e/integration/costs.bats | 51 --------- test/e2e/integration/plan.bats | 32 ++++++ 8 files changed, 157 insertions(+), 78 deletions(-) delete mode 100644 test/e2e/integration/costs.bats diff --git a/.gitignore b/.gitignore index 635fa12c2..3f265f2c4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.orig .idea .vscode +.vim gcp-auth.json /*.env #local dev artifacts diff --git a/pkg/controller/configuration/ensure.go b/pkg/controller/configuration/ensure.go index 7b08d9d61..9f66c02c7 100644 --- a/pkg/controller/configuration/ensure.go +++ b/pkg/controller/configuration/ensure.go @@ -570,37 +570,65 @@ func (c *Controller) ensureCostStatus(configuration *terraformv1alphav1.Configur return reconcile.Result{}, nil } + input := secret.Data["costs.json"] + if len(input) == 0 { + return reconcile.Result{}, nil + } + // @step: parse the cost report json - if secret.Data["costs.json"] != nil { - report := make(map[string]interface{}) - if err := json.NewDecoder(bytes.NewReader(secret.Data["costs.json"])).Decode(&report); err != nil { - cond.Failed(err, "Failed to decode the terraform costs report") + values := map[string]float64{ + "totalMonthlyCost": 0, + "totalHourlyCost": 0, + } + for key := range values { + value := gjson.GetBytes(input, key) + if !value.Exists() { + cond.ActionRequired("Cost report does not include the %s value", key) - return reconcile.Result{}, err + return reconcile.Result{}, controller.ErrIgnore } - var monthly, hourly float64 + cost, err := strconv.ParseFloat(value.String(), 64) + if err != nil { + cond.ActionRequired("Cost report contains an include value: %q for item: %s", value.String(), key) - if v, ok := report["totalMonthlyCost"].(string); ok { - if x, err := strconv.ParseFloat(v, 64); err == nil { - monthly = x - } - } - if v, ok := report["totalHourlyCost"].(string); ok { - if x, err := strconv.ParseFloat(v, 64); err == nil { - hourly = x - } + return reconcile.Result{}, controller.ErrIgnore } + values[key] = cost + } - configuration.Status.Costs = &terraformv1alphav1.CostStatus{ - Enabled: true, - Hourly: fmt.Sprintf("$%v", hourly), - Monthly: fmt.Sprintf("$%v", monthly), - } + configuration.Status.Costs = &terraformv1alphav1.CostStatus{ + Enabled: true, + Hourly: fmt.Sprintf("$%v", values["totalHourlyCost"]), + Monthly: fmt.Sprintf("$%v", values["totalMonthlyCost"]), + } - // @step: update the prometheus metrics - monthlyCostMetric.WithLabelValues(labels...).Set(monthly) - hourlyCostMetric.WithLabelValues(labels...).Set(hourly) + // @step: update the prometheus metrics + monthlyCostMetric.WithLabelValues(labels...).Set(values["totalMonthlyCost"]) + hourlyCostMetric.WithLabelValues(labels...).Set(values["totalHourlyCost"]) + + // @step: copy the infracost report into the configuration namespace + copied := &v1.Secret{} + copied.Namespace = configuration.GetNamespace() + copied.Name = configuration.GetTerraformCostSecretName() + copied.Labels = map[string]string{ + terraformv1alphav1.ConfigurationNameLabel: configuration.GetName(), + terraformv1alphav1.ConfigurationUIDLabel: string(configuration.GetUID()), + } + copied.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: terraformv1alphav1.SchemeGroupVersion.String(), + Kind: terraformv1alphav1.ConfigurationKind, + Name: configuration.GetName(), + UID: configuration.GetUID(), + }, + } + copied.Data = secret.Data + + if err := kubernetes.CreateOrForceUpdate(ctx, c.cc, copied); err != nil { + cond.Failed(err, "Failed to create or update the terraform costs secret") + + return reconcile.Result{}, err } return reconcile.Result{}, nil @@ -610,6 +638,7 @@ func (c *Controller) ensureCostStatus(configuration *terraformv1alphav1.Configur // ensurePolicyStatus is responsible for checking the checkov results and refusing to continue if failed func (c *Controller) ensurePolicyStatus(configuration *terraformv1alphav1.Configuration, state *state) controller.EnsureFunc { cond := controller.ConditionMgr(configuration, terraformv1alphav1.ConditionTerraformPolicy, c.recorder) + key := "results_json.json" return func(ctx context.Context) (reconcile.Result, error) { switch { @@ -637,7 +666,7 @@ func (c *Controller) ensurePolicyStatus(configuration *terraformv1alphav1.Config } // @step: retrieve summary from the report - checksFailed := gjson.GetBytes(secret.Data["results_json.json"], "summary.failed") + checksFailed := gjson.GetBytes(secret.Data[key], "summary.failed") if !checksFailed.Exists() { cond.Failed(errors.New("missing report"), "Security report does not contain a summary of finding, please contact platform administrator") @@ -650,6 +679,30 @@ func (c *Controller) ensurePolicyStatus(configuration *terraformv1alphav1.Config return reconcile.Result{}, controller.ErrIgnore } + // @step: copy the report into the configuration namespace + copied := &v1.Secret{} + copied.Namespace = configuration.GetNamespace() + copied.Name = configuration.GetTerraformPolicySecretName() + copied.Labels = map[string]string{ + terraformv1alphav1.ConfigurationNameLabel: configuration.GetName(), + terraformv1alphav1.ConfigurationUIDLabel: string(configuration.GetUID()), + } + copied.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: terraformv1alphav1.SchemeGroupVersion.String(), + Kind: terraformv1alphav1.ConfigurationKind, + Name: configuration.GetName(), + UID: configuration.GetUID(), + }, + } + copied.Data = secret.Data + + if err := kubernetes.CreateOrForceUpdate(ctx, c.cc, copied); err != nil { + cond.Failed(err, "Failed to create or update the terraform policy secret") + + return reconcile.Result{}, err + } + if checksFailed.Int() > 0 { cond.ActionRequired("Configuration has failed security policy, refusing to continue") diff --git a/pkg/controller/configuration/reconcile_test.go b/pkg/controller/configuration/reconcile_test.go index 697c057c8..7cf3014e6 100644 --- a/pkg/controller/configuration/reconcile_test.go +++ b/pkg/controller/configuration/reconcile_test.go @@ -680,6 +680,19 @@ var _ = Describe("Configuration Controller", func() { Expect(configuration.Status.Costs.Monthly).To(Equal("$100")) Expect(configuration.Status.Costs.Hourly).To(Equal("$0.01")) }) + + It("should have copied the secret into the configuration namespace", func() { + expected := "\n{\n\t\"totalHourlyCost\": \"0.01\",\n \"totalMonthlyCost\": \"100.00\"\n}\n" + secret := &v1.Secret{} + secret.Namespace = configuration.Namespace + secret.Name = configuration.GetTerraformCostSecretName() + found, err := kubernetes.GetIfExists(context.TODO(), cc, secret) + + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(secret.Data).To(HaveKey("costs.json")) + Expect(string(secret.Data["costs.json"])).To(Equal(expected)) + }) }) }) @@ -1447,6 +1460,17 @@ terraform { Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred()) Expect(len(list.Items)).To(Equal(1)) }) + + It("should copied the report into the configuration namespace", func() { + secret := &v1.Secret{} + secret.Namespace = configuration.Namespace + secret.Name = configuration.GetTerraformPolicySecretName() + found, err := kubernetes.GetIfExists(context.TODO(), ctrl.cc, secret) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(secret.Data).To(HaveKey("results_json.json")) + Expect(string(secret.Data["results_json.json"])).To(ContainSubstring("summary")) + }) }) When("policy report contains no failed checks", func() { @@ -1483,6 +1507,17 @@ terraform { Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred()) Expect(len(list.Items)).To(Equal(2)) }) + + It("should copied the report into the configuration namespace", func() { + secret := &v1.Secret{} + secret.Namespace = configuration.Namespace + secret.Name = configuration.GetTerraformPolicySecretName() + found, err := kubernetes.GetIfExists(context.TODO(), ctrl.cc, secret) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(secret.Data).To(HaveKey("results_json.json")) + Expect(string(secret.Data["results_json.json"])).To(ContainSubstring("summary")) + }) }) }) }) diff --git a/test/e2e/check-suite.sh b/test/e2e/check-suite.sh index cc539bd0c..32a7f770e 100755 --- a/test/e2e/check-suite.sh +++ b/test/e2e/check-suite.sh @@ -55,7 +55,6 @@ run_checks() { "${UNITS}/cloud/${CLOUD}/provider.bats" "${UNITS}/cloud/${CLOUD}/plan.bats" "${UNITS}/plan.bats" - "${UNITS}/costs.bats" "${UNITS}/apply.bats" "${UNITS}/cloud/${CLOUD}/confirm.bats" "${UNITS}/drift.bats" diff --git a/test/e2e/integration/checkov.bats b/test/e2e/integration/checkov.bats index 615a61542..b18f28ba9 100644 --- a/test/e2e/integration/checkov.bats +++ b/test/e2e/integration/checkov.bats @@ -104,6 +104,15 @@ EOF [[ "$status" -eq 0 ]] } +@test "We should have a copy the policy report in the configuration namespace" { + UUID=$(kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json | jq -r '.metadata.uid') + [[ "$status" -eq 0 ]] + runit "kubectl -n ${APP_NAMESPACE} get secret policy-${UUID}" + [[ "$status" -eq 0 ]] + runit "kubectl -n ${APP_NAMESPACE} get secret policy-${UUID} -o json" "jq -r '.data.results_json.json'" + [[ "$status" -eq 0 ]] +} + @test "We should see the conditions indicate the configuration failed policy" { POD=$(kubectl -n ${APP_NAMESPACE} get pod -l terraform.appvia.io/configuration=${RESOURCE_NAME} -l terraform.appvia.io/stage=plan -o json | jq -r '.items[0].metadata.name') [[ "$status" -eq 0 ]] diff --git a/test/e2e/integration/cloud/aws/infracost.bats b/test/e2e/integration/cloud/aws/infracost.bats index 61f2d964e..d37a34b17 100644 --- a/test/e2e/integration/cloud/aws/infracost.bats +++ b/test/e2e/integration/cloud/aws/infracost.bats @@ -18,7 +18,7 @@ load ../../../lib/helper setup() { - [[ ! -f ${BATS_PARENT_TMPNAME}.skip ]] || skip "skip remaining tests" + [[ ! -f "${BATS_PARENT_TMPNAME}.skip" ]] || skip "skip remaining tests" } teardown() { @@ -26,7 +26,8 @@ teardown() { } @test "We should have a token for the infracost integration" { - [[ -z ${INFRACOST_API_KEY} ]] && touch ${BATS_PARENT_TMPNAME}.skip + [[ -z "${INFRACOST_API_KEY}" ]] && touch ${BATS_PARENT_TMPNAME}.skip + [[ "$status" -eq 0 ]] } @test "We should be able to create a configuration which costs money on aws" { diff --git a/test/e2e/integration/costs.bats b/test/e2e/integration/costs.bats deleted file mode 100644 index e13ff4f26..000000000 --- a/test/e2e/integration/costs.bats +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bats -# -# Copyright 2021 Appvia Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load ../lib/helper - -setup() { - [[ ! -f ${BATS_PARENT_TMPNAME}.skip ]] || skip "skip remaining tests" -} - -teardown() { - [[ -n "$BATS_TEST_COMPLETED" ]] || touch ${BATS_PARENT_TMPNAME}.skip -} - -@test "We should have a token for the infracost api" { - [[ -z "$INFRACOST_API_KEY" ]] || touch ${BATS_PARENT_TMPNAME}.skip - [[ "$status" -eq 0 ]] -} - -@test "We should have a secret in the terraform namespace containing the report" { - UUID=$(kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json | jq -r '.metadata.uid') - [[ "$status" -eq 0 ]] - - runit "kubectl -n ${NAMESPACE} get secret costs-${UUID}" - [[ "$status" -eq 0 ]] -} - -@test "We should see the cost integration is enabled" { - runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.enabled' | grep -q true" - [[ "$status" -eq 0 ]] -} - -@test "We should see the cost associated to the configuration" { - runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.monthly' | grep -q '\$0'" - [[ "$status" -eq 0 ]] - runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.hourly' | grep -q '\$0'" - [[ "$status" -eq 0 ]] -} diff --git a/test/e2e/integration/plan.bats b/test/e2e/integration/plan.bats index ca385eb0e..71b0f70b5 100644 --- a/test/e2e/integration/plan.bats +++ b/test/e2e/integration/plan.bats @@ -85,3 +85,35 @@ teardown() { runit "kubectl -n ${APP_NAMESPACE} logs ${POD} 2>&1" "grep -q '\[build\] completed'" [[ "$status" -eq 0 ]] } + +@test "We should have a secret in the terraform namespace containing the report" { + UUID=$(kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json | jq -r '.metadata.uid') + [[ "$status" -eq 0 ]] + + runit "kubectl -n ${NAMESPACE} get secret costs-${UUID}" + [[ "$status" -eq 0 ]] + runit "kubectl -n ${NAMESPACE} get secret costs-${UUID} -o json" "jq -r '.data[\"costs.json\"]'" + [[ "$status" -eq 0 ]] +} + +@test "We should see the cost integration is enabled" { + runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.enabled' | grep -q true" + [[ "$status" -eq 0 ]] +} + +@test "We should see the cost associated to the configuration" { + runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.monthly' | grep -q '\$0'" + [[ "$status" -eq 0 ]] + runit "kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json" "jq -r '.status.costs.hourly' | grep -q '\$0'" + [[ "$status" -eq 0 ]] +} + +@test "We should have a copy of the infracost report in the configuration namespace" { + UUID=$(kubectl -n ${APP_NAMESPACE} get configuration ${RESOURCE_NAME} -o json | jq -r '.metadata.uid') + [[ "$status" -eq 0 ]] + + runit "kubectl -n ${APP_NAMESPACE} get secret costs-${UUID}" + [[ "$status" -eq 0 ]] + runit "kubectl -n ${APP_NAMESPACE} get secret costs-${UUID} -o json" "jq -r '.data[\"costs.json\"]'" + [[ "$status" -eq 0 ]] +}