KEP-17: Pipe task implementation

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/containerd/containerd v1.2.9 // indirect
+	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v1.4.2-0.20190916154449-92cc603036dd
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
@@ -38,6 +39,7 @@ require (
 	github.com/stretchr/testify v1.4.0
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca
 	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339 // indirect
 	golang.org/x/tools v0.0.0-20191025023517-2077df36852e // indirect
 	gopkg.in/yaml.v2 v2.2.4

go.sum

@@ -21,6 +21,7 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
@@ -60,6 +61,7 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
 github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/containerd v1.2.9 h1:6tyNjBmAMG47QuFPIT9LgiiexoVxC6qpTGR+eD0R0Z8=
@@ -75,9 +77,11 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw=
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
@@ -99,10 +103,12 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e h1:bRcq7ruHMqCVB/ugLbBylx+LrccNACFDEaqAD/aZ80Q=
 github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -111,6 +117,7 @@ github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -392,6 +399,7 @@ github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFW
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -468,6 +476,7 @@ github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
@@ -613,6 +622,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

keps/0017-pipe-tasks.md

@@ -43,21 +43,20 @@ Allowing to pipe all kind of files (>1Mb) between tasks requires a general-purpo
 
 ## Proposal
 
-This section describes how pipe tasks and files they produce are configured in the operator. Here is a task definition that produces a file that will be stored as a Secret, referenced by `{{.Pipes.Certificate}}` key:
+This section describes how pipe tasks and files they produce are configured in the operator. Here is a pipe task definition that produces a file that will be stored as a Secret:
 ```yaml
 tasks:
   - name: gencert
     kind: Pipe
     spec:
-      containerSpec:
-        ...
+      container: cert-container.yaml
       pipe:
-        file: /usr/share/MyKey.key
-        kind: Secret # ConfigMap
-        key: {{.Pipes.Certificate}}
+        - file: /usr/share/MyKey.key
+          kind: Secret # ConfigMap
+          key: Mycertificate
 ``` 
 
-`containerSpec` field is described in detail below. `pipe` field defines how the produced file will be stored and referenced. The key `{{.Pipes.Certificate}}` value will be generated by KUDO to avoid collisions and to make sure that it is not used before the file is generated. In the above example we would generate a secret name like `instance-myapp.deploy.bootstrap.gencert.pipes.certificate-#hash` to capture instance name along with plan/phase/step/task of the secret origin and the hash of its content. KUDO would use this hash to avoid recreating secrets for unchanged files on a plan rerun. We would also use labels ensuring that the secret is cleaned up when the corresponding Instance is removed. 
+`container` field is described in detail below. `key` will be used by the resources referencing generated file as following: `{{.Pipes.Mycertificate}}`. In the above example we would create a secret named `instance-myapp.deploy.bootstrap.gencert.mycertificate` to capture instance name along with plan/phase/step/task of the secret origin. The secret name would be stable so that a user can rerun the certificate generating task and reload all pods using it. Pipe file name will be used as Secret/ConfigMap data key. We would also use labels ensuring that the secret is cleaned up when the corresponding Instance is removed. `pipe` field is an array and can define how multiple generated files are stored and referenced.
 
 The corresponding `gencert` task can be used as usual in e.g.:
 ```yaml
@@ -75,28 +74,28 @@ plans:
 
 Note that piped files has to be generated before they can be used. In the example above, `bootstrap` phase has a strategy `serial` so that certificate generated in the `gencert` step can be used in subsequent steps. Or stated differently resources can not reference piped secrets/configmaps generated within the same step or within a parallel series of steps (it has to be a different step in the phase with serial strategy or a different phase). 
 
-For the pipe task `containerSpec`, we allow a [ContainerSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#container-v1-core) to be specified. Reasons for that are explained below in the implementation details. The ContainerSpec has to define a shared volume where the files are stored.
+For the pipe task `container`, we allow a [core/v1 container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#container-v1-core) resource to be specified. Reasons for that are explained below in the implementation details. The container has to define a volumeMount where the files are stored.
 
 ```yaml
-containerSpec:
-  volumes:
-  - name: shared-data
-    emptyDir: {}
+volumes:
+- name: shared-data
+  emptyDir: {}
 
-  containers:
-    - name: gencert
-      image: frapsoft/openssl
-      imagePullPolicy: Always
-      command: [ "sh", "-c" ]
-      args:
-        - openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out MyCertificate.crt -keyout /usr/share/MyKey.key
-      volumeMounts:
-        - name: shared-data
-          mountPath: /usr/share/
+containers:
+  - name: gencert
+    image: frapsoft/openssl
+    imagePullPolicy: Always
+    command: [ "sh", "-c" ]
+    args:
+      - openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out MyCertificate.crt -keyout /usr/share/MyKey.key
+    volumeMounts:
+      - name: shared-data
+        mountPath: /usr/share/
 ```
 
-Any subsequent step resource (if the phase strategy is `serial`) might reference previously generated file by its key `{{.Pipes.Certificate}}` e.g.:
+Any subsequent step resource (if the phase strategy is `serial`) might reference previously generated file by its key e.g.:
 ```yaml
+# some Pod spec referenced in a subsequent step
 spec:
   containers:
   - name: myapp
@@ -107,18 +106,18 @@ spec:
   volumes:
   - name: cert
     secret:
-      secretName: {{.Pipes.Certificate}}
+      secretName: {{.Pipes.Mycertificate}}
 ```
 
 ### Limitations
-- File generating container has to be side-effect free (meaning side-effects that are observable outside of the container like a 3rd party API call) as the container might be executed multiple times on failure. If that's not the case, `restartPolicy: Never` has to be set which would prevent the task from finishing successfully.
+- File generating container has to be side-effect free (meaning side-effects that are observable outside of the container like a 3rd party API call) as the container might be executed multiple times on failure. If that's not the case, `restartPolicy: OnFailure` has to be set to ensure the pod would be restarted if there is an issue with the initContainer.
 - Only files <1Mb are applicable to be stored as ConfigMap or Secret.
 - Only one resource per pipe task is allowed. If needed multiple tasks must be used.
 
 ### Implementation Details/Notes/Constraints
 
 There are several ways to implement pipe tasks, each one having its challenges and complexities. The approach below allows us not to worry about Pod container life-cycle as well as keep the storing logic in the KUDO controller:
-- Provided ContainerSpec is injected into a Pod as an [InitContainer](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/). This is the simplest way to wait for container completion. This is also the reason why pipe task resource definition is a ContainerSpec and not a complete Pod specification. Pod init container can not have Lifecycle actions, Readiness probes, or Liveness probes fields defined. 
+- Provided container is injected into a Pod as an [InitContainer](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/). This is the simplest way to wait for container completion. This is also the reason why pipe task resource definition is a container and not a complete Pod specification. Pod init container can not have Lifecycle actions, Readiness probes, or Liveness probes fields defined. 
 - The main container is a `busybox` image, running the `sleep infinity` command, which purpose is to wait for KUDO to extract and store the files.
 - Pod status `READY: 1/1, STATUS: Running` means that the init container has run successfully. As this point KUDO can copy out referenced files using `kubectl cp` and store them as specified.
 - Once files are stored, KUDO can delete the Pod and proceed to the next task.
@@ -134,7 +133,7 @@ spec:
   - name: shared-data
     emptyDir: {}
 
-  # Inject provided ContainerSpec generating /tmp/foo.txt
+  # Inject provided container generating /tmp/foo.txt
   initContainers:
     - name: init
       image: busybox
@@ -165,7 +164,7 @@ foo-bar-bazz
 
 ## Alternatives
 
-An alternative approach would use the provided ContainerSpec as the main container (or let the user provide a complete Pod spec). We would inject a sidecar with a go executable which would:
+An alternative approach would use the provided `container` as the main container (or let the user provide a complete Pod spec). We would inject a sidecar with a go executable which would:
 - Use controller runtime, watch its own Pod status and wait for termination of the main container
 - Once the main container exits, it would copy the referenced files and store them as specified
 - We would use `restartPolicy: Never/OnFailure` to prevent the main container from restarting

pkg/apis/kudo/v1beta1/operatorversion_types.go

@@ -121,6 +121,7 @@ type Task struct {
 type TaskSpec struct {
 	ResourceTaskSpec
 	DummyTaskSpec
+	PipeTaskSpec
 }
 
 // ResourceTaskSpec is referencing a list of resources
@@ -135,6 +136,19 @@ type DummyTaskSpec struct {
 	Done    bool `json:"done"`
 }
 
+// PipeTask specifies a task that generates files and stores them for later usage in subsequent tasks
+type PipeTaskSpec struct {
+	Container string     `json:"container"`
+	Pipe      []PipeSpec `json:"pipe"`
+}
+
+// PipeSpec describes how a file generated by a PipeTask is stored and referenced
+type PipeSpec struct {
+	File string `json:"file"`
+	Kind string `json:"kind"`
+	Key  string `json:"key"`
+}
+
 // OperatorVersionStatus defines the observed state of OperatorVersion.
 type OperatorVersionStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster

pkg/engine/health/health.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"reflect"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	kudov1beta1 "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
@@ -67,6 +68,12 @@ func IsHealthy(obj runtime.Object) error {
 		}
 		return fmt.Errorf("instance's active plan is in state %v", obj.Status.AggregatedStatus.Status)
 
+	case *corev1.Pod:
+		if obj.Status.Phase == corev1.PodRunning {
+			return nil
+		}
+		return fmt.Errorf("pod \"%v\" is not running yet: %s", obj.Name, obj.Status.Phase)
+
 	// unless we build logic for what a healthy object is, assume it's healthy when created.
 	default:
 		log.Printf("HealthUtil: Unknown type %s is marked healthy by default", reflect.TypeOf(obj))