Merge branch 'master' into avolkov/SDK/Components---Stopped-allowing-…

…the-user-specify-output-paths-when-calling-task-factory

Gopkg.toml

@@ -24,6 +24,13 @@
 #   go-tests = true
 #   unused-packages = true
 
+required = [
+  "k8s.io/code-generator/cmd/defaulter-gen",
+  "k8s.io/code-generator/cmd/deepcopy-gen",
+  "k8s.io/code-generator/cmd/client-gen",
+  "k8s.io/code-generator/cmd/lister-gen",
+  "k8s.io/code-generator/cmd/informer-gen",
+]
 
 [[constraint]]
   name = "github.com/argoproj/argo"
@@ -117,10 +124,6 @@
   branch = "release-1.9"
   name = "k8s.io/apimachinery"
 
-[[constraint]]
-  branch = "release-6.0"
-  name = "k8s.io/client-go"
-
 [[constraint]]
   name = "k8s.io/kubernetes"
   version = "1.11.1"
@@ -129,6 +132,10 @@
   go-tests = true
   unused-packages = true
 
+   [[prune.project]]
+    name = "k8s.io/code-generator"
+    unused-packages = false
+
 [[override]]
   branch = "release-1.11"
   name = "k8s.io/apimachinery"
@@ -138,9 +145,15 @@
   name = "k8s.io/api"
 
 [[override]]
-  branch = "release-8.0"
   name = "k8s.io/client-go"
+  # revision for tag "kubernetes-1.11.1"
+  revision = "59698c7d9724b0f95f9dc9e7f7dfdcc3dfeceb82"
 
 [[override]]
   name = "github.com/spf13/pflag"
   version = "1.0.1"
+
+[[override]]
+  name = "k8s.io/code-generator"
+  # revision for tag "kubernetes-1.11.1"
+  revision = "6702109cc68eb6fe6350b83e14407c8d7309fd1a"

README.md

@@ -15,7 +15,7 @@ The Kubeflow pipelines service has the following goals:
 
 ## Documentation
 
-Get started with your first pipeline and read further information in the [documentation](https://github.com/kubeflow/pipelines/wiki).
+Get started with your first pipeline and read further information in the [Kubeflow Pipelines documentation](https://www.kubeflow.org/docs/guides/pipelines/pipelines-overview/).
 
 ## Blog posts
 

backend/Dockerfile

@@ -31,7 +31,16 @@ RUN pip3 install /kfp.tar.gz
 
 WORKDIR /samples
 COPY ./samples .
-RUN  find . -maxdepth 2 -name "*.py" -exec dsl-compile --py {} --output {}.tar.gz \;
+
+#We need to check that all samples have been compiled without error.
+#For find program, the -exec argument is a filter predicate just like -name. It only affects whether the file is "found", not the find's exit code.
+#One way to solve this problem is to check whether we have any python pipelines that cannot compile. Here the exit code is the number of such files:
+#RUN  bash -e -c 'exit $(find . -maxdepth 2 -name "*.py" ! -exec dsl-compile --py {} --output {}.tar.gz \; -print | wc -l)'
+#I think it's better to just use a shell loop though.
+#RUN  for pipeline in $(find . -maxdepth 2 -name '*.py' -type f); do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
+#The "for" loop breaks on all whitespace, so we either need to override IFS or use the "read" command instead.
+RUN  find . -maxdepth 2 -name '*.py' -type f | while read pipeline; do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
+
 
 FROM alpine
 

backend/src/apiserver/interceptor.go

@@ -26,13 +26,14 @@ import (
 // to be executed before and after all API handler calls, e.g. Logging, error handling.
 // For more details, see https://github.com/grpc/grpc-go/blob/master/interceptor.go
 func apiServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
-	glog.Infof("%v called", info.FullMethod)
+	glog.Infof("%v handler starting", info.FullMethod)
 	resp, err = handler(ctx, req)
 	if err != nil {
 		util.LogError(util.Wrapf(err, "%s call failed", info.FullMethod))
 		// Convert error to gRPC errors
 		err = util.ToGRPCError(err)
 		return
 	}
+	glog.Infof("%v handler finished", info.FullMethod)
 	return
 }

backend/src/crd/README.md

@@ -1,13 +1,12 @@
 ## Scheduled Workflow CRD/controller
 
-### How to generate the API client code from the API specification? 
+### How to generate the API client code from the API specification?
 
 Get the dependencies:
 
 ```
+dep ensure
 go get -u ./...
-go get -u k8s.io/client-go/...
-go get -u k8s.io/code-generator/...
 ```
 
 Generate the API client code from the API specification:
@@ -16,11 +15,11 @@ Generate the API client code from the API specification:
 ./hack/update-codegen.sh
 ```
 
-### How to run the ScheduledWorkflow controller from the command line? 
+### How to run the ScheduledWorkflow controller from the command line?
 
 The following assumes that your Kubernetes configuration file is located at '$HOME/.kube/config'.
 
-To create the resource for the CRD, execute: 
+To create the resource for the CRD, execute:
 
 ```
 kubectl create -f ./install/manifests/scheduledworkflow-crd.yaml
@@ -32,7 +31,7 @@ Output:
 customresourcedefinition.apiextensions.k8s.io "scheduledworkflows.kubeflow.org" created
 ```
 
-To run the controller locally, execute: 
+To run the controller locally, execute:
 configuration file
 
 ```
@@ -47,19 +46,19 @@ Started workers
 Wait for shut down
 ```
 
-To run a sample workflow on a schedule, execute: 
+To run a sample workflow on a schedule, execute:
 
 ```
 kubectl create -f ./samples/scheduledworkflow/every-minute-cron.yaml
 ```
 
-Output: 
+Output:
 
 ```
 scheduledworkflow.kubeflow.org "every-minute-cron" created
 ```
 
-To see the current list of ScheduledWorkflows, execute: 
+To see the current list of ScheduledWorkflows, execute:
 
 ```
 kubectl get swf
@@ -72,7 +71,7 @@ NAME                AGE
 every-minute-cron   1m
 ```
 
-To see the current status of the ScheduledWorklfow named 'every-minute-cron', execute: 
+To see the current status of the ScheduledWorklfow named 'every-minute-cron', execute:
 
 ```
 kubectl describe swf every-minute-cron
@@ -89,7 +88,7 @@ Annotations:  <none>
 API Version:  kubeflow.org/v1alpha1
 Kind:         ScheduledWorkflow
 Metadata:
-  Cluster Name:        
+  Cluster Name:
   Creation Timestamp:  2018-06-06T01:24:55Z
   Generation:          0
   Initializers:        <nil>
@@ -116,7 +115,7 @@ Spec:
           Command:
             cowsay
           Image:  docker/whalesay
-          Name:   
+          Name:
           Resources:
         Inputs:
           Parameters:
@@ -150,4 +149,4 @@ Status:
       UID:           b0b63a82-692a-11e8-9fd5-42010a8a0021
       [...]
 ```
-### 
+###

backend/src/crd/hack/update-codegen.sh

@@ -21,12 +21,13 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})
 echo "SCRIPT_ROOT is $SCRIPT_ROOT"
-CODEGEN_PKG=${SCRIPT_ROOT}/../../../../../../k8s.io/code-generator
+CODEGEN_PKG=${SCRIPT_ROOT}/../../../../vendor/k8s.io/code-generator
 echo "CODEGEN_PKG is $CODEGEN_PKG"
 
 ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
-  github.com/kubeflow/pipelines/backend/src/crd/pkg/client github.com/kubeflow/pipelines/backend/src/crd/pkg/apis \
+  github.com/kubeflow/pipelines/backend/src/crd/pkg/client \
+  github.com/kubeflow/pipelines/backend/src/crd/pkg/apis \
   scheduledworkflow:v1alpha1 \
-  --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt
+  --go-header-file ${SCRIPT_ROOT}/custom-boilerplate.go.txt