[FEATURE] - Additional Controller Secrets (#204)

* [FEATURE] - Additional Controller Secrets

Currently unless you override the template there is no way of the platform team getting
additional secrets in the execution of the terraform jobs. With this PR we've added
a command line options --additional-secret which is always added into the executor
terraform container.

* - updating the chart to include the value and the ability to set the
    additional executor secrets on the command line via values.yaml

charts/terraform-controller/templates/deployment.yaml

@@ -63,6 +63,9 @@ spec:
             - --enable-watchers={{ .Values.controller.enableWatchers }}
             - --enable-webhook={{ .Values.controller.webhooks.enabled }}
             - --executor-image={{ .Values.controller.images.executor }}
+            {{- range .Values.controller.executorSecrets }}
+            - --executor-secret={{ . }}
+            {{- end }}
             - --infracost-image={{ .Values.controller.images.infracost }}
             - --metrics-port={{ .Values.controller.metricsPort }}
             - --policy-image={{ .Values.controller.images.policy }}

charts/terraform-controller/values.yaml

@@ -16,6 +16,9 @@ controller:
   namespace: default
   # Indicates if the controller should register its own CRDs
   registerCRDs: true
+  # Executor secrets includes the following secrets in 'all' execution jobs. The secret is added
+  # as an environment variables (spec.envFrom) into the terraform container of the executor
+  executorSecrets: []
   # Configuration related to costs
   costs:
     # Name of the secret containing the infracost api token

cmd/controller/main.go

@@ -64,6 +64,7 @@ func main() {
 	flags.IntVar(&config.APIServerPort, "apiserver-port", 10080, "The port the apiserver should be listening on")
 	flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
 	flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")
+	flags.StringSliceVar(&config.ExecutorSecrets, "executor-secret", []string{}, "Name of a secret in controller namespace which should be added to the job")
 	flags.StringVar(&config.ExecutorImage, "executor-image", "ghcr.io/appvia/terraform-executor:latest", "The image to use for the executor")
 	flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts")
 	flags.StringVar(&config.InfracostsSecretName, "cost-secret", "", "Name of the secret on the controller namespace containing your infracost token")

pkg/assets/job.yaml.tpl

@@ -159,11 +159,16 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-        {{- if eq .Provider.Source "secret" }}
         envFrom:
+        {{- if eq .Provider.Source "secret" }}
           - secretRef:
               name: {{ .Provider.SecretRef.Name }}
         {{- end }}
+        {{- range .ExecutorSecrets }}
+          - secretRef:
+              name: {{ . }}
+              optional: true
+        {{- end }}
         resources:
           limits:
             cpu: 1

pkg/controller/configuration/controller.go

@@ -60,6 +60,10 @@ type Controller struct {
 	cache *cache.Cache
 	// recorder is the kubernetes event recorder
 	recorder record.EventRecorder
+	// ExecutorSecrets is a collection of secrets which should be added to the
+	// executors job everytime - these are configured by the platform team on the
+	// cli options
+	ExecutorSecrets []string
 	// ControllerNamespace is the namespace where the runner is running
 	ControllerNamespace string
 	// EnableInfracosts enables the cost analytics via infracost
@@ -85,11 +89,12 @@ type Controller struct {
 // Add is called to setup the manager for the controller
 func (c *Controller) Add(mgr manager.Manager) error {
 	log.WithFields(log.Fields{
-		"enable_costs":    c.EnableInfracosts,
-		"enable_watchers": c.EnableWatchers,
-		"namespace":       c.ControllerNamespace,
-		"policy_image":    c.PolicyImage,
-		"terraform_image": c.TerraformImage,
+		"additional_secrets": len(c.ExecutorSecrets),
+		"enable_costs":       c.EnableInfracosts,
+		"enable_watchers":    c.EnableWatchers,
+		"namespace":          c.ControllerNamespace,
+		"policy_image":       c.PolicyImage,
+		"terraform_image":    c.TerraformImage,
 	}).Info("adding the configuration controller")
 
 	switch {

pkg/controller/configuration/delete.go

@@ -77,6 +77,7 @@ func (c *Controller) ensureTerraformDestroy(configuration *terraformv1alphav1.Co
 		runner, err := batch.NewTerraformDestroy(jobs.Options{
 			EnableInfraCosts: c.EnableInfracosts,
 			ExecutorImage:    c.ExecutorImage,
+			ExecutorSecrets:  c.ExecutorSecrets,
 			InfracostsImage:  c.InfracostsImage,
 			InfracostsSecret: c.InfracostsSecretName,
 			Namespace:        c.ControllerNamespace,

pkg/controller/configuration/ensure.go

@@ -457,11 +457,12 @@ func (c *Controller) ensureTerraformPlan(configuration *terraformv1alphav1.Confi
 			AdditionalLabels: map[string]string{terraformv1alphav1.DriftAnnotation: configuration.GetAnnotations()[terraformv1alphav1.DriftAnnotation]},
 			EnableInfraCosts: c.EnableInfracosts,
 			ExecutorImage:    c.ExecutorImage,
+			ExecutorSecrets:  c.ExecutorSecrets,
 			InfracostsImage:  c.InfracostsImage,
 			InfracostsSecret: c.InfracostsSecretName,
 			Namespace:        c.ControllerNamespace,
-			PolicyImage:      c.PolicyImage,
 			PolicyConstraint: state.checkovConstraint,
+			PolicyImage:      c.PolicyImage,
 			Template:         state.jobTemplate,
 			TerraformImage:   GetTerraformImage(configuration, c.TerraformImage),
 		}
@@ -820,6 +821,7 @@ func (c *Controller) ensureTerraformApply(configuration *terraformv1alphav1.Conf
 		runner, err := jobs.New(configuration, state.provider).NewTerraformApply(jobs.Options{
 			EnableInfraCosts: c.EnableInfracosts,
 			ExecutorImage:    c.ExecutorImage,
+			ExecutorSecrets:  c.ExecutorSecrets,
 			InfracostsImage:  c.InfracostsImage,
 			InfracostsSecret: c.InfracostsSecretName,
 			Namespace:        c.ControllerNamespace,

pkg/controller/configuration/reconcile_test.go

@@ -874,6 +874,57 @@ var _ = Describe("Configuration Controller", func() {
 		})
 	})
 
+	// ADDITIONAL SECRETS
+	When("the controller has been configured with additional secrets", func() {
+		BeforeEach(func() {
+			configuration = fixtures.NewValidBucketConfiguration(cfgNamespace, "bucket")
+			Setup(configuration)
+
+			ctrl.ExecutorSecrets = []string{"secret1", "secret2"}
+			result, _, rerr = controllertests.Roll(context.TODO(), ctrl, configuration, 3)
+		})
+
+		It("should have the conditions", func() {
+			Expect(cc.Get(context.TODO(), configuration.GetNamespacedName(), configuration)).ToNot(HaveOccurred())
+			Expect(configuration.Status.Conditions).To(HaveLen(defaultConditions))
+		})
+
+		It("should indicate the failure on the conditions", func() {
+			Expect(cc.Get(context.TODO(), configuration.GetNamespacedName(), configuration)).ToNot(HaveOccurred())
+
+			cond := configuration.Status.GetCondition(terraformv1alphav1.ConditionProviderReady)
+			Expect(cond.Status).To(Equal(metav1.ConditionTrue))
+			Expect(cond.Reason).To(Equal(corev1alphav1.ReasonReady))
+			Expect(cond.Message).To(Equal("Provider ready"))
+		})
+
+		It("should have create a plan", func() {
+			list := &batchv1.JobList{}
+
+			Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred())
+			Expect(len(list.Items)).To(Equal(1))
+		})
+
+		It("should have the additional secrets added", func() {
+			list := &batchv1.JobList{}
+			Expect(cc.List(context.TODO(), list, client.InNamespace(ctrl.ControllerNamespace))).ToNot(HaveOccurred())
+			Expect(len(list.Items)).To(Equal(1))
+
+			job := list.Items[0]
+			Expect(job.Spec.Template.Spec.Containers).To(HaveLen(1))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom).To(HaveLen(3))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[0].SecretRef.Name).To(Equal("aws"))
+
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Name).To(Equal("secret1"))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Optional).ToNot(BeNil())
+			Expect(*job.Spec.Template.Spec.Containers[0].EnvFrom[1].SecretRef.Optional).To(BeTrue())
+
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Name).To(Equal("secret2"))
+			Expect(job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Optional).ToNot(BeNil())
+			Expect(*job.Spec.Template.Spec.Containers[0].EnvFrom[2].SecretRef.Optional).To(BeTrue())
+		})
+	})
+
 	When("configuration has not yet run the terraform plan", func() {
 		BeforeEach(func() {
 			configuration = fixtures.NewValidBucketConfiguration(cfgNamespace, "bucket")

pkg/server/server.go

@@ -147,6 +147,7 @@ func New(cfg *rest.Config, config Config) (*Server, error) {
 		EnableTerraformVersions: config.EnableTerraformVersions,
 		EnableWatchers:          config.EnableWatchers,
 		ExecutorImage:           config.ExecutorImage,
+		ExecutorSecrets:         config.ExecutorSecrets,
 		InfracostsImage:         config.InfracostsImage,
 		InfracostsSecretName:    config.InfracostsSecretName,
 		JobTemplate:             config.JobTemplate,

pkg/server/types.go

@@ -21,6 +21,8 @@ import "time"
 
 // Config is the configuration for the controller
 type Config struct {
+	// ExecutorSecrets is a list of additional secrets to be added to the executor
+	ExecutorSecrets []string
 	// APIServerPort is the port to listen on
 	APIServerPort int
 	// DriftControllerInterval is the interval for the controller to check for drift

pkg/utils/jobs/jobs.go

@@ -45,6 +45,8 @@ type Options struct {
 	EnableInfraCosts bool
 	// ExecutorImage is the image to use for the terraform jobs
 	ExecutorImage string
+	// ExecutorSecrets is a list of additional secrets to add to the job
+	ExecutorSecrets []string
 	// InfracostsImage is the image to use for infracosts
 	InfracostsImage string
 	// InfracostsSecret is the name of the secret contain the infracost token and url
@@ -190,6 +192,7 @@ func (r *Render) createTerraformFromTemplate(options Options, stage string) (*ba
 		},
 		"EnableInfraCosts":   options.EnableInfraCosts,
 		"EnableVariables":    r.configuration.HasVariables(),
+		"ExecutorSecrets":    options.ExecutorSecrets,
 		"ImagePullPolicy":    "IfNotPresent",
 		"Policy":             options.PolicyConstraint,
 		"ServiceAccount":     DefaultServiceAccount,