Merge e22f047 into 9b77d4a

.cloudbuild.yaml

@@ -165,11 +165,11 @@ steps:
 # Build the local pipeline component images
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/local/containers/confusion_matrix && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildConfusionMatrix'
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/local/containers/roc && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildROC'
 
 # Build the tagged samples

Gopkg.toml

@@ -24,6 +24,13 @@
 #   go-tests = true
 #   unused-packages = true
 
+required = [
+  "k8s.io/code-generator/cmd/defaulter-gen",
+  "k8s.io/code-generator/cmd/deepcopy-gen",
+  "k8s.io/code-generator/cmd/client-gen",
+  "k8s.io/code-generator/cmd/lister-gen",
+  "k8s.io/code-generator/cmd/informer-gen",
+]
 
 [[constraint]]
   name = "github.com/argoproj/argo"
@@ -117,10 +124,6 @@
   branch = "release-1.9"
   name = "k8s.io/apimachinery"
 
-[[constraint]]
-  branch = "release-6.0"
-  name = "k8s.io/client-go"
-
 [[constraint]]
   name = "k8s.io/kubernetes"
   version = "1.11.1"
@@ -129,6 +132,10 @@
   go-tests = true
   unused-packages = true
 
+   [[prune.project]]
+    name = "k8s.io/code-generator"
+    unused-packages = false
+
 [[override]]
   branch = "release-1.11"
   name = "k8s.io/apimachinery"
@@ -138,9 +145,15 @@
   name = "k8s.io/api"
 
 [[override]]
-  branch = "release-8.0"
   name = "k8s.io/client-go"
+  # revision for tag "kubernetes-1.11.1"
+  revision = "59698c7d9724b0f95f9dc9e7f7dfdcc3dfeceb82"
 
 [[override]]
   name = "github.com/spf13/pflag"
   version = "1.0.1"
+
+[[override]]
+  name = "k8s.io/code-generator"
+  # revision for tag "kubernetes-1.11.1"
+  revision = "6702109cc68eb6fe6350b83e14407c8d7309fd1a"

README.md

@@ -15,12 +15,7 @@ The Kubeflow pipelines service has the following goals:
 
 ## Documentation
 
-Get started with your first pipeline and read further information in the [Kubeflow Pipelines documentation](https://www.kubeflow.org/docs/guides/pipelines/pipelines-overview/).
-
-## Blog posts
-
-* [Getting started with Kubeflow Pipelines](https://cloud.google.com/blog/products/ai-machine-learning/getting-started-kubeflow-pipelines) (By Amy Unruh)
-* [How to create and deploy a Kubeflow Machine Learning Pipeline](https://towardsdatascience.com/how-to-create-and-deploy-a-kubeflow-machine-learning-pipeline-part-1-efea7a4b650f) (By Lak Lakshmanan)
+Get started with your first pipeline and read further information in the [documentation](https://github.com/kubeflow/pipelines/wiki).
 
 ## Acknowledgments
 

backend/Dockerfile

@@ -14,33 +14,14 @@ COPY . .
 RUN apk add --update gcc musl-dev
 RUN go build -o /bin/apiserver backend/src/apiserver/*.go
 
-FROM python:3.5 as compiler
-
-RUN apt-get update -y && \
-    apt-get install --no-install-recommends -y -q default-jdk wget 
-
-RUN pip3 install setuptools==40.5.0
-
-RUN wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar
-
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
-WORKDIR /go/src/github.com/kubeflow/pipelines/sdk/python
-RUN ./build.sh /kfp.tar.gz
-RUN pip3 install /kfp.tar.gz
+FROM python:3.5.0-slim as compiler
 
+# This is hard coded to 0.0.26. Once kfp DSK release process is automated,
+# we can dynamically refer to the version from same commit SHA.
+RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade
 WORKDIR /samples
 COPY ./samples .
-
-#We need to check that all samples have been compiled without error.
-#For find program, the -exec argument is a filter predicate just like -name. It only affects whether the file is "found", not the find's exit code.
-#One way to solve this problem is to check whether we have any python pipelines that cannot compile. Here the exit code is the number of such files:
-#RUN  bash -e -c 'exit $(find . -maxdepth 2 -name "*.py" ! -exec dsl-compile --py {} --output {}.tar.gz \; -print | wc -l)'
-#I think it's better to just use a shell loop though.
-#RUN  for pipeline in $(find . -maxdepth 2 -name '*.py' -type f); do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
-#The "for" loop breaks on all whitespace, so we either need to override IFS or use the "read" command instead.
-RUN  find . -maxdepth 2 -name '*.py' -type f | while read pipeline; do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
-
+RUN  find . -maxdepth 2 -name "*.py" -exec dsl-compile --py {} --output {}.tar.gz \;
 
 FROM alpine
 

backend/src/apiserver/interceptor.go

@@ -26,14 +26,13 @@ import (
 // to be executed before and after all API handler calls, e.g. Logging, error handling.
 // For more details, see https://github.com/grpc/grpc-go/blob/master/interceptor.go
 func apiServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
-	glog.Infof("%v handler starting", info.FullMethod)
+	glog.Infof("%v called", info.FullMethod)
 	resp, err = handler(ctx, req)
 	if err != nil {
 		util.LogError(util.Wrapf(err, "%s call failed", info.FullMethod))
 		// Convert error to gRPC errors
 		err = util.ToGRPCError(err)
 		return
 	}
-	glog.Infof("%v handler finished", info.FullMethod)
 	return
 }

backend/src/crd/README.md

@@ -1,13 +1,12 @@
 ## Scheduled Workflow CRD/controller
 
-### How to generate the API client code from the API specification? 
+### How to generate the API client code from the API specification?
 
 Get the dependencies:
 
 ```
+dep ensure
 go get -u ./...
-go get -u k8s.io/client-go/...
-go get -u k8s.io/code-generator/...
 ```
 
 Generate the API client code from the API specification:
@@ -16,11 +15,11 @@ Generate the API client code from the API specification:
 ./hack/update-codegen.sh
 ```
 
-### How to run the ScheduledWorkflow controller from the command line? 
+### How to run the ScheduledWorkflow controller from the command line?
 
 The following assumes that your Kubernetes configuration file is located at '$HOME/.kube/config'.
 
-To create the resource for the CRD, execute: 
+To create the resource for the CRD, execute:
 
 ```
 kubectl create -f ./install/manifests/scheduledworkflow-crd.yaml
@@ -32,7 +31,7 @@ Output:
 customresourcedefinition.apiextensions.k8s.io "scheduledworkflows.kubeflow.org" created
 ```
 
-To run the controller locally, execute: 
+To run the controller locally, execute:
 configuration file
 
 ```
@@ -47,19 +46,19 @@ Started workers
 Wait for shut down
 ```
 
-To run a sample workflow on a schedule, execute: 
+To run a sample workflow on a schedule, execute:
 
 ```
 kubectl create -f ./samples/scheduledworkflow/every-minute-cron.yaml
 ```
 
-Output: 
+Output:
 
 ```
 scheduledworkflow.kubeflow.org "every-minute-cron" created
 ```
 
-To see the current list of ScheduledWorkflows, execute: 
+To see the current list of ScheduledWorkflows, execute:
 
 ```
 kubectl get swf
@@ -72,7 +71,7 @@ NAME                AGE
 every-minute-cron   1m
 ```
 
-To see the current status of the ScheduledWorklfow named 'every-minute-cron', execute: 
+To see the current status of the ScheduledWorklfow named 'every-minute-cron', execute:
 
 ```
 kubectl describe swf every-minute-cron
@@ -89,7 +88,7 @@ Annotations:  <none>
 API Version:  kubeflow.org/v1alpha1
 Kind:         ScheduledWorkflow
 Metadata:
-  Cluster Name:        
+  Cluster Name:
   Creation Timestamp:  2018-06-06T01:24:55Z
   Generation:          0
   Initializers:        <nil>
@@ -116,7 +115,7 @@ Spec:
           Command:
             cowsay
           Image:  docker/whalesay
-          Name:   
+          Name:
           Resources:
         Inputs:
           Parameters:
@@ -150,4 +149,4 @@ Status:
       UID:           b0b63a82-692a-11e8-9fd5-42010a8a0021
       [...]
 ```
-### 
+###

backend/src/crd/hack/update-codegen.sh

@@ -21,12 +21,13 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})
 echo "SCRIPT_ROOT is $SCRIPT_ROOT"
-CODEGEN_PKG=${SCRIPT_ROOT}/../../../../../../k8s.io/code-generator
+CODEGEN_PKG=${SCRIPT_ROOT}/../../../../vendor/k8s.io/code-generator
 echo "CODEGEN_PKG is $CODEGEN_PKG"
 
 ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
-  github.com/kubeflow/pipelines/backend/src/crd/pkg/client github.com/kubeflow/pipelines/backend/src/crd/pkg/apis \
+  github.com/kubeflow/pipelines/backend/src/crd/pkg/client \
+  github.com/kubeflow/pipelines/backend/src/crd/pkg/apis \
   scheduledworkflow:v1alpha1 \
-  --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt
+  --go-header-file ${SCRIPT_ROOT}/custom-boilerplate.go.txt