[Sample] CI Sample: helloworld

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/README.md

@@ -0,0 +1,17 @@
+# Hello World CI Sample
+
+## Overview
+
+This sample uses cloudbuild to implement the continuous integration process of a simple pipeline that outputs "hello world" to the console. Once all set up, you can push your code to github repo, then the build process in cloud build will be triggered automatically, then a run will be created in kubeflow pipeline. You can view your pipeline and the run in kubeflow pipelines. 
+
+Besides, we use **REST API** to call kubeflow pipeline to create a new version and a run in this sample. Other methods to create pipeline version can be found in mnist sample in this repo, i.e., use kfp SDK.
+
+## Usage
+
+To use this pipeline, you need to:
+
+* Set up a trigger in cloud build that connects to your github repo.
+* Replace the constants to your own configuration in cloudbuild.yaml
+* Replace images in the pipeline.py to your own images (the ones you built in cloudbuild.yaml)
+* Set your cloud registry public accessible
+* Set your bucket public accessible, or authenticate cloudbuild to cloud storage

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/cloudbuild.yaml

@@ -0,0 +1,59 @@
+steps:
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      [
+        "build",
+        "-t",
+        "${_GCR_PATH}/helloworld-ci:$COMMIT_SHA",
+        "-t",
+        "${_GCR_PATH}/helloworld-ci:latest",
+        "--cache-from",
+        "${_GCR_PATH}/helloworld-ci:latest",
+        "${_CODE_PATH}/helloworld",
+      ]
+    id: "BuildImages"
+  - name: "python:3.7-slim"
+    entrypoint: "/bin/sh"
+    args: [
+        "-c",
+        "cd ${_CODE_PATH};
+        pip3 install cffi==1.12.3 --upgrade;
+        pip3 install kfp;
+        sed -i s/image: helloworld-ci/image: ${_GCR_PATH}/helloworld-ci:$COMMIT_SHA/g component.yaml;
+        python pipeline.py;
+        cp pipeline.py.zip /workspace/pipeline.zip",
+      ]
+    id: "PackagePipeline"
+
+  - name: "gcr.io/cloud-builders/gsutil"
+    args:
+      [
+        "cp",
+        "/workspace/pipeline.zip",
+        "${_GS_BUCKET}/$COMMIT_SHA/pipeline.zip",
+      ]
+    id: "UploadPipeline"
+    waitFor: ["PackagePipeline"]
+
+
+  - name: "gcr.io/cloud-builders/curl"
+    entrypoint: "/bin/sh"
+    args: 
+      [
+        "-c",
+        "curl.bash $COMMIT_SHA ${_PIPELINE_ID} ${_GS_BUCKET} ${_PIPELINE_ENDPOINT}"
+      ]
+    id: "CreatePipelineVersionAndRun"
+
+images:
+  - "${_GCR_PATH}/helloworld-ci:$COMMIT_SHA"
+  - "${_GCR_PATH}/helloworld-ci:latest"
+
+substitutions:
+  _GCR_PATH: [Your cloud registry path. For example, gcr.io/myproject]
+  _CODE_PATH: /workspace/hello-world
+  _NAMESPACE: kubeflow
+  _PIPELINE_ID: [Your kubeflow pipeline id to create a version on. Get it from kfp UI.]
+  _GS_BUCKET: [Name of your cloud storage bucket. For example, 'gs://my-bucket']
+  _PIPELINE_ENDPOINT: [Your exposed pipeline endpoint 'ml-pipeline'. You can expose it in k8s workloads on gcp.
+                       For example: https://0.0.0.1:8888]

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/component.yaml

@@ -0,0 +1,5 @@
+name: helloworld-ci
+description: Print "hello world" to console
+implementation:
+  container:
+    image: helloworld-ci

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/curl.bash

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# strip gs:// prefix
+bucket_name=$(echo $3 | sed 's/gs:\/\///')
+data='{"name":'\""ci-$1"\"', "code_source_url": "https://github.com/kubeflow/pipelines/tree/'"$1"'", "package_url": {"pipeline_url": "https://storage.googleapis.com/'"$bucket_name"'/'"$1"'/pipeline.zip"}, 
+"resource_references": [{"key": {"id": '\""$2"\"', "type":3}, "relationship":1}]}'
+
+version=$(curl -H "Content-Type: application/json" -X POST -d "$data" "$4"/apis/v1beta1/pipeline_versions | jq -r ".id")
+
+# create run
+rundata='{"name":'\""$1-run"\"', 
+"resource_references": [{"key": {"id": '\""$version"\"', "type":4}, "relationship":2}]}'
+echo "$rundata"
+curl -H "Content-Type: application/json" -X POST -d "$rundata" "$4"/apis/v1beta1/runs

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/helloworld/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3
+COPY helloworld.py .
+CMD ["python", "./helloworld.py"]

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/helloworld/helloworld.py

@@ -0,0 +1,5 @@
+def main():
+  print("hello world!")
+
+if __name__ == "__main__":
+  main()

samples/contrib/versioned-pipeline-ci-samples/helloworld-ci-sample/pipeline.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import kfp
+import kfp.dsl as dsl
+
+@dsl.pipeline(
+    name='hello world pipeline sample',
+    description='A simple sample using curl to interact with kfp'
+)
+def helloworld_ci_pipeline():
+  import os
+  train_op = kfp.components.load_component_from_file('./component.yaml')
+  train = train_op()
+
+if __name__ == '__main__':
+  import kfp.compiler as compiler
+  compiler.Compiler().compile(helloworld_ci_pipeline, __file__ + '.zip')