[FEATURE] - Additional Controller Secrets

Currently unless you override the template there is no way of the platform team getting additional secrets in the execution of the terraform jobs. With this PR we've added a command line options --additional-secret which is always added into the executor terraform container.
appvia · Jul 3, 2022 · 09836c7 · 09836c7
1 parent 48960b0
commit 09836c7
Show file tree

Hide file tree

Showing 9 changed files with 103 additions and 32 deletions.
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -64,6 +64,7 @@ func main() {
 	flags.IntVar(&config.APIServerPort, "apiserver-port", 10080, "The port the apiserver should be listening on")
 	flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
 	flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")
+	flags.StringVar(&config.AdditionalSecrets, "additional-secret", []string{}, "Name of a secret in controller namespace which should be added to the job")
 	flags.StringVar(&config.ExecutorImage, "executor-image", "ghcr.io/appvia/terraform-executor:latest", "The image to use for the executor")
 	flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts")
 	flags.StringVar(&config.InfracostsSecretName, "cost-secret", "", "Name of the secret on the controller namespace containing your infracost token")

diff --git a/pkg/assets/job.yaml.tpl b/pkg/assets/job.yaml.tpl
@@ -159,11 +159,16 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-        {{- if eq .Provider.Source "secret" }}
         envFrom:
+        {{- if eq .Provider.Source "secret" }}
           - secretRef:
               name: {{ .Provider.SecretRef.Name }}
         {{- end }}
+        {{- range .AdditionalSecrets }}
+          - secretRef:
+              name: {{ . }}
+              optional: true
+        {{- end }}
         resources:
           limits:
             cpu: 1

diff --git a/pkg/controller/configuration/controller.go b/pkg/controller/configuration/controller.go
@@ -60,6 +60,10 @@ type Controller struct {
 	cache *cache.Cache
 	// recorder is the kubernetes event recorder
 	recorder record.EventRecorder
+	// AdditionalSecrets is a collection of secrets which should be added to the
+	// executors job everytime - these are configured by the platform team on the
+	// cli options
+	AdditionalSecrets []string
 	// ControllerNamespace is the namespace where the runner is running
 	ControllerNamespace string
 	// EnableInfracosts enables the cost analytics via infracost
@@ -85,11 +89,12 @@ type Controller struct {
 // Add is called to setup the manager for the controller
 func (c *Controller) Add(mgr manager.Manager) error {
 	log.WithFields(log.Fields{
-		"enable_costs":    c.EnableInfracosts,
-		"enable_watchers": c.EnableWatchers,
-		"namespace":       c.ControllerNamespace,
-		"policy_image":    c.PolicyImage,
-		"terraform_image": c.TerraformImage,
+		"additional_secrets": len(c.AdditionalSecrets),
+		"enable_costs":       c.EnableInfracosts,
+		"enable_watchers":    c.EnableWatchers,
+		"namespace":          c.ControllerNamespace,
+		"policy_image":       c.PolicyImage,
+		"terraform_image":    c.TerraformImage,
 	}).Info("adding the configuration controller")
 
 	switch {

diff --git a/pkg/controller/configuration/delete.go b/pkg/controller/configuration/delete.go
@@ -75,13 +75,14 @@ func (c *Controller) ensureTerraformDestroy(configuration *terraformv1alphav1.Co
 		// @step: generate the destroy job
 		batch := jobs.New(configuration, state.provider)
 		runner, err := batch.NewTerraformDestroy(jobs.Options{
-			EnableInfraCosts: c.EnableInfracosts,
-			ExecutorImage:    c.ExecutorImage,
-			InfracostsImage:  c.InfracostsImage,
-			InfracostsSecret: c.InfracostsSecretName,
-			Namespace:        c.ControllerNamespace,
-			Template:         state.jobTemplate,
-			TerraformImage:   GetTerraformImage(configuration, c.TerraformImage),
+			AdditionalSecrets: c.AdditionalSecrets,
+			EnableInfraCosts:  c.EnableInfracosts,
+			ExecutorImage:     c.ExecutorImage,
+			InfracostsImage:   c.InfracostsImage,
+			InfracostsSecret:  c.InfracostsSecretName,
+			Namespace:         c.ControllerNamespace,
+			Template:          state.jobTemplate,
+			TerraformImage:    GetTerraformImage(configuration, c.TerraformImage),
 		})
 		if err != nil {
 			cond.Failed(err, "Failed to create the terraform destroy job")

diff --git a/pkg/controller/configuration/ensure.go b/pkg/controller/configuration/ensure.go
@@ -454,16 +454,17 @@ func (c *Controller) ensureTerraformPlan(configuration *terraformv1alphav1.Confi
 
 		// @step: lets build the options to render the job
 		options := jobs.Options{
-			AdditionalLabels: map[string]string{terraformv1alphav1.DriftAnnotation: configuration.GetAnnotations()[terraformv1alphav1.DriftAnnotation]},
-			EnableInfraCosts: c.EnableInfracosts,
-			ExecutorImage:    c.ExecutorImage,
-			InfracostsImage:  c.InfracostsImage,
-			InfracostsSecret: c.InfracostsSecretName,
-			Namespace:        c.ControllerNamespace,
-			PolicyImage:      c.PolicyImage,
-			PolicyConstraint: state.checkovConstraint,
-			Template:         state.jobTemplate,
-			TerraformImage:   GetTerraformImage(configuration, c.TerraformImage),
+			AdditionalSecrets: c.AdditionalSecrets,
+			AdditionalLabels:  map[string]string{terraformv1alphav1.DriftAnnotation: configuration.GetAnnotations()[terraformv1alphav1.DriftAnnotation]},
+			EnableInfraCosts:  c.EnableInfracosts,
+			ExecutorImage:     c.ExecutorImage,
+			InfracostsImage:   c.InfracostsImage,
+			InfracostsSecret:  c.InfracostsSecretName,
+			Namespace:         c.ControllerNamespace,
+			PolicyImage:       c.PolicyImage,
+			PolicyConstraint:  state.checkovConstraint,
+			Template:          state.jobTemplate,
+			TerraformImage:    GetTerraformImage(configuration, c.TerraformImage),
 		}
 
 		// @step: use the options to generate the job
@@ -818,13 +819,14 @@ func (c *Controller) ensureTerraformApply(configuration *terraformv1alphav1.Conf
 
 		// @step: create the terraform job
 		runner, err := jobs.New(configuration, state.provider).NewTerraformApply(jobs.Options{
-			EnableInfraCosts: c.EnableInfracosts,
-			ExecutorImage:    c.ExecutorImage,
-			InfracostsImage:  c.InfracostsImage,
-			InfracostsSecret: c.InfracostsSecretName,
-			Namespace:        c.ControllerNamespace,
-			Template:         state.jobTemplate,
-			TerraformImage:   GetTerraformImage(configuration, c.TerraformImage),
+			AdditionalSecrets: c.AdditionalSecrets,
+			EnableInfraCosts:  c.EnableInfracosts,
+			ExecutorImage:     c.ExecutorImage,
+			InfracostsImage:   c.InfracostsImage,
+			InfracostsSecret:  c.InfracostsSecretName,
+			Namespace:         c.ControllerNamespace,
+			Template:          state.jobTemplate,
+			TerraformImage:    GetTerraformImage(configuration, c.TerraformImage),
 		})
 		if err != nil {
 			cond.Failed(err, "Failed to create the terraform apply job")

diff --git a/pkg/controller/configuration/reconcile_test.go b/pkg/controller/configuration/reconcile_test.go
@@ -874,6 +874,57 @@ var _ = Describe("Configuration Controller", func() {
 		})
 	})
 
+	// ADDITIONAL SECRETS
+	When("the controller has been configured with additional secrets", func() {
+		BeforeEach(func() {
+			configuration = fixtures.NewValidBucketConfiguration(cfgNamespace, "bucket")
+			Setup(configuration)
+
+			ctrl.AdditionalSecrets = []string{"secret1", "secret2"}
+			result, _, rerr = controllertests.Roll(context.TODO(), ctrl, configuration, 3)
+		})
+
+		It("should have the conditions", func() {
+			Expect(cc.Get(context.TODO(), configuration.GetNamespacedName(), configuration)).ToNot(HaveOccurred())
+			Expect(configuration.Status.Conditions).To(HaveLen(defaultConditions))
+		})
+
+		It("should indicate the failure on the conditions", func() {
+			Expect(cc.Get(context.TODO(), configuration.GetNamespacedName(), configuration)).ToNot(HaveOccurred())
+
+			cond := configuration.Status.GetCondition(terraformv1alphav1.ConditionProviderReady)
+			Expect(cond.Status).To(Equal(metav1.ConditionTrue))
+			Expect(cond.Reason).To(Equal(corev1alphav1.ReasonReady))
+			Expect(cond.Message).To(Equal("Provider ready"))
+		})
+
+		It("should have create a plan", func() {
+			list := &batchv1.JobList{}
+
+			Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred())
+			Expect(len(list.Items)).To(Equal(1))
+		})
+
+		It("should have the additional secrets added", func() {
+			list := &batchv1.JobList{}
+			Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred())
+			Expect(len(list.Items)).To(Equal(1))
+
+			job := list.Items[0]
+			Expect(job.Spec.Template.Spec.Containers).To(HaveLen(1))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom).To(HaveLen(3))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[0].SecretRef.Name).To(Equal("aws"))
+
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Name).To(Equal("secret1"))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Optional).ToNot(BeNil())
+			Expect(*job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Optional).To(BeTrue())
+
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Name).To(Equal("secret2"))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Optional).ToNot(BeNil())
+			Expect(*job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Optional).To(BeTrue())
+		})
+	})
+
 	When("configuration has not yet run the terraform plan", func() {
 		BeforeEach(func() {
 			configuration = fixtures.NewValidBucketConfiguration(cfgNamespace, "bucket")

diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -142,6 +142,7 @@ func New(cfg *rest.Config, config Config) (*Server, error) {
 	}
 
 	if err := (&configuration.Controller{
+		AdditionalSecrets:       config.AdditionalSecrets,
 		ControllerNamespace:     config.Namespace,
 		EnableInfracosts:        (config.InfracostsSecretName != ""),
 		EnableTerraformVersions: config.EnableTerraformVersions,

diff --git a/pkg/server/types.go b/pkg/server/types.go
@@ -21,6 +21,8 @@ import "time"
 
 // Config is the configuration for the controller
 type Config struct {
+	// AdditionalSecrets is a list of additional secrets to be added to the executor
+	AdditionalSecrets []string
 	// APIServerPort is the port to listen on
 	APIServerPort int
 	// DriftControllerInterval is the interval for the controller to check for drift

diff --git a/pkg/utils/jobs/jobs.go b/pkg/utils/jobs/jobs.go
@@ -39,6 +39,8 @@ const DefaultServiceAccount = "terraform-executor"
 
 // Options is the configuration for the render
 type Options struct {
+	// AdditionalSecrets is a list of additional secrets to add to the job
+	AdditionalSecrets []string
 	// AdditionalLabels are additional labels added to the job
 	AdditionalLabels map[string]string
 	// EnableInfraCosts is the flag to enable cost analysis
@@ -172,8 +174,9 @@ func (r *Render) createTerraformFromTemplate(options Options, stage string) (*ba
 	}
 
 	params := map[string]interface{}{
-		"GenerateName": fmt.Sprintf("%s-%s-", r.configuration.Name, stage),
-		"Namespace":    options.Namespace,
+		"AdditionalSecrets": options.AdditionalSecrets,
+		"GenerateName":      fmt.Sprintf("%s-%s-", r.configuration.Name, stage),
+		"Namespace":         options.Namespace,
 		"Labels": utils.MergeStringMaps(map[string]string{
 			terraformv1alphav1.ConfigurationGenerationLabel: fmt.Sprintf("%d", r.configuration.GetGeneration()),
 			terraformv1alphav1.ConfigurationNameLabel:       r.configuration.GetName(),