Merge pull request #20 from mikebaum/add-env-property

Add env property to pipeline API

README.md

@@ -19,17 +19,19 @@ kind: Pipeline
 metadata:
   name: pipeline-sample
 spec:
-  replicas: 2
+  replicas: 1
   config:
     input:
-      broker:
-        inputs:
-          - file:
-              paths: ["./config/meow.txt"]
-          - generate:
-              mapping: root = "woof"
-              interval: 10s
-              count: 0
+      generate:
+        mapping: |
+          let favorite_animal = env("FAVORITE_ANIMAL") 
+          root = match $favorite_animal {
+            "cat" => file("/config/cat.txt")
+            "dog" => file("/config/dog.txt")
+            _ => file("/config/dog.txt")
+          }
+        interval: 5s
+        count: 0
 
     pipeline:
       processors:
@@ -39,8 +41,14 @@ spec:
       stdout: {}
 
   configFiles:
-    meow.txt: |
+    cat.txt: |
       meow
+    dog.txt: |
+      woof
+
+  env:
+    - name: FAVORITE_ANIMAL
+      value: cat
 ```
 
 Once the resource is deployed, you can monitor the state of the resource:

api/v1alpha1/pipeline_types.go

@@ -1,6 +1,7 @@
 package v1alpha1
 
 import (
+	v1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -29,6 +30,9 @@ type PipelineSpec struct {
 	// ConfigFiles Additional configuration, as Key/Value pairs, that will be mounted as files with the /config
 	// directory on the pod. The key should be the file name and the value should be its content.
 	ConfigFiles map[string]string `json:"configFiles,omitempty"`
+
+	// Env Environment Variables to set in the benthos pipeline pod.
+	Env []v1.EnvVar `json:"env,omitempty"`
 }
 
 // PipelineStatus defines the observed state of Pipeline

config/crd/bases/captain.benthos.dev_pipelines.yaml

@@ -68,8 +68,121 @@ spec:
                   type: string
                 description: |-
                   ConfigFiles Additional configuration, as Key/Value pairs, that will be mounted as files with the /config
-                  directory on the pod. The key should be the file name and the value should be it's content.
+                  directory on the pod. The key should be the file name and the value should be its content.
                 type: object
+              env:
+                description: Env Environment Variable to set in the benthos pipeline
+                  pod.
+                items:
+                  description: EnvVar represents an environment variable present in
+                    a Container.
+                  properties:
+                    name:
+                      description: Name of the environment variable. Must be a C_IDENTIFIER.
+                      type: string
+                    value:
+                      description: |-
+                        Variable references $(VAR_NAME) are expanded
+                        using the previously defined environment variables in the container and
+                        any service environment variables. If a variable cannot be resolved,
+                        the reference in the input string will be unchanged. Double $$ are reduced
+                        to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.
+                        "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
+                        Escaped references will never be expanded, regardless of whether the variable
+                        exists or not.
+                        Defaults to "".
+                      type: string
+                    valueFrom:
+                      description: Source for the environment variable's value. Cannot
+                        be used if value is not empty.
+                      properties:
+                        configMapKeyRef:
+                          description: Selects a key of a ConfigMap.
+                          properties:
+                            key:
+                              description: The key to select.
+                              type: string
+                            name:
+                              description: |-
+                                Name of the referent.
+                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                TODO: Add other useful fields. apiVersion, kind, uid?
+                              type: string
+                            optional:
+                              description: Specify whether the ConfigMap or its key
+                                must be defined
+                              type: boolean
+                          required:
+                          - key
+                          type: object
+                          x-kubernetes-map-type: atomic
+                        fieldRef:
+                          description: |-
+                            Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
+                            spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
+                          properties:
+                            apiVersion:
+                              description: Version of the schema the FieldPath is
+                                written in terms of, defaults to "v1".
+                              type: string
+                            fieldPath:
+                              description: Path of the field to select in the specified
+                                API version.
+                              type: string
+                          required:
+                          - fieldPath
+                          type: object
+                          x-kubernetes-map-type: atomic
+                        resourceFieldRef:
+                          description: |-
+                            Selects a resource of the container: only resources limits and requests
+                            (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
+                          properties:
+                            containerName:
+                              description: 'Container name: required for volumes,
+                                optional for env vars'
+                              type: string
+                            divisor:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              description: Specifies the output format of the exposed
+                                resources, defaults to "1"
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            resource:
+                              description: 'Required: resource to select'
+                              type: string
+                          required:
+                          - resource
+                          type: object
+                          x-kubernetes-map-type: atomic
+                        secretKeyRef:
+                          description: Selects a key of a secret in the pod's namespace
+                          properties:
+                            key:
+                              description: The key of the secret to select from.  Must
+                                be a valid secret key.
+                              type: string
+                            name:
+                              description: |-
+                                Name of the referent.
+                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                TODO: Add other useful fields. apiVersion, kind, uid?
+                              type: string
+                            optional:
+                              description: Specify whether the Secret or its key must
+                                be defined
+                              type: boolean
+                          required:
+                          - key
+                          type: object
+                          x-kubernetes-map-type: atomic
+                      type: object
+                  required:
+                  - name
+                  type: object
+                type: array
               image:
                 description: Image defines the image and tag to use for the Benthos
                   deployment.

config/samples/captain_v1alpha1_pipeline.yaml

@@ -12,14 +12,16 @@ spec:
   replicas: 1
   config:
     input:
-      broker:
-        inputs:
-          - file:
-              paths: ["./config/meow.txt"]
-          - generate:
-              mapping: root = "woof"
-              interval: 10s
-              count: 0
+      generate:
+        mapping: |
+          let favorite_animal = env("FAVORITE_ANIMAL") 
+          root = match $favorite_animal {
+            "cat" => file("/config/cat.txt")
+            "dog" => file("/config/dog.txt")
+            _ => file("/config/dog.txt")
+          }
+        interval: 5s
+        count: 0
 
     pipeline:
       processors:
@@ -29,5 +31,11 @@ spec:
       stdout: {}
 
   configFiles:
-    meow.txt: |
+    cat.txt: |
       meow
+    dog.txt: |
+      woof
+
+  env:
+    - name: FAVORITE_ANIMAL
+      value: cat

internal/pkg/resource/resource.go

@@ -54,6 +54,7 @@ func NewDeployment(name string, namespace string, scope captainv1.PipelineSpec)
 								ReadOnly:  true,
 							},
 						},
+						Env: scope.Env,
 					}},
 					Volumes: []corev1.Volume{
 						{