Refactor presubmit-tests-with-pipeline-deployment.sh to run in other …

…projects (#1732)

* Refactor presubmit-tests-with-pipeline-deployment.sh so that it can be run from a different project

* Simplify getting service account from cluster.

* Copy image builder image instead of granting permission

* Add missed yes command

* fix stuff

* Let other usages of image-builder image become configurable

* let test workflow use image builder image

test/README.md

@@ -65,6 +65,21 @@ However you can keep them by providing additional parameter.
 argo submit integration_test_gke.yaml -p branch="my-branch" -p cleanup="false"
 ```
 
+### Run presubmit-tests-with-pipeline-deployment.sh locally
+
+Run the following commands from root of kubeflow/pipelines repo.
+```
+#$PULL_PULL_SHA and $WORKSPACE are env variables set by Prow
+export PULL_PULL_SHA=pull-sha-placeholder
+export WORKSPACE=$(pwd) # root of kubeflow/pipelines git repo
+
+./test/presubmit-tests-with-pipeline-deployment.sh \
+  --workflow_file e2e_test_gke_v2.yaml \ # You can specify other workflows you want to test too.
+  --test_result_folder ${FOLDER_NAME_TO_HOLD_TEST_RESULT} \
+  --test_result_bucket ${YOUR_GCS_TEST_RESULT_BUCKET} \
+  --project ${YOUR_GCS_PROJECT}
+```
+
 ## Troubleshooting
 
 **Q: Why is my test taking so long on GKE?**

test/build_image.yaml

@@ -25,6 +25,8 @@ spec:
   arguments:
     parameters:
     - name: image-build-context-gcs-uri
+    - name: image-builder-image
+      value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
     - name: api-image
     - name: frontend-image
     - name: scheduledworkflow-image
@@ -105,7 +107,7 @@ spec:
           valueFrom:
             path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
+      image: "{{workflow.parameters.image-builder-image}}"
       imagePullPolicy: 'Always'
       args: [
         "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",

test/component_test.yaml

@@ -25,6 +25,8 @@ spec:
   arguments:
     parameters:
     - name: image-build-context-gcs-uri
+    - name: image-builder-image
+      value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
     - name: commit-sha
     - name: component-image-prefix
     - name: target-image-prefix
@@ -158,7 +160,7 @@ spec:
           valueFrom:
             path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
+      image: "{{workflow.parameters.image-builder-image}}"
       imagePullPolicy: 'Always'
       args: [
         "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",

test/deploy-kubeflow.sh

@@ -25,7 +25,8 @@ KS_VERSION="0.13.0"
 curl -LO https://github.com/ksonnet/ksonnet/releases/download/v${KS_VERSION}/ks_${KS_VERSION}_linux_amd64.tar.gz
 tar -xzf ks_${KS_VERSION}_linux_amd64.tar.gz
 chmod +x ./ks_${KS_VERSION}_linux_amd64/ks
-mv ./ks_${KS_VERSION}_linux_amd64/ks /usr/local/bin/
+# Add ks to PATH
+PATH=$PATH:`pwd`/ks_${KS_VERSION}_linux_amd64
 
 ## Download latest kubeflow release source code
 KUBEFLOW_SRC=${DIR}/kubeflow_latest_release

test/e2e_test_gke_v2.yaml

@@ -25,6 +25,8 @@ spec:
   arguments:
     parameters:
     - name: image-build-context-gcs-uri
+    - name: image-builder-image
+      value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
     - name: target-image-prefix
     - name: test-results-gcs-dir
     - name: initialization-test-image-suffix
@@ -196,7 +198,7 @@ spec:
           valueFrom:
             path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
+      image: "{{workflow.parameters.image-builder-image}}"
       imagePullPolicy: 'Always'
       args: [
         "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",

test/presubmit-tests-with-pipeline-deployment.sh

@@ -20,6 +20,7 @@ usage()
 {
     echo "usage: deploy.sh
     [--platform             the deployment platform. Valid values are: [gcp, minikube]. Default is gcp.]
+    [--project              the gcp project. Default is ml-pipeline-test. Only used when platform is gcp.]
     [--workflow_file        the file name of the argo workflow to run]
     [--test_result_bucket   the gcs bucket that argo workflow store the result to. Default is ml-pipeline-test
     [--test_result_folder   the gcs folder that argo workflow store the result to. Always a relative directory to gs://<gs_bucket>/[PULL_SHA]]
@@ -30,7 +31,6 @@ usage()
 PLATFORM=gcp
 PROJECT=ml-pipeline-test
 TEST_RESULT_BUCKET=ml-pipeline-test
-GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_PULL_SHA}
 TIMEOUT_SECONDS=1800
 NAMESPACE=kubeflow
 
@@ -39,6 +39,9 @@ while [ "$1" != "" ]; do
              --platform )             shift
                                       PLATFORM=$1
                                       ;;
+             --project )              shift
+                                      PROJECT=$1
+                                      ;;
              --workflow_file )        shift
                                       WORKFLOW_FILE=$1
                                       ;;
@@ -61,6 +64,7 @@ while [ "$1" != "" ]; do
 done
 
 # Variables
+GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/${PULL_PULL_SHA}
 TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/${TEST_RESULT_FOLDER}
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
 
@@ -73,10 +77,22 @@ source "${DIR}/deploy-kubeflow.sh"
 # Install Argo CLI and test-runner service account
 source "${DIR}/install-argo.sh"
 
+IMAGE_BUILDER_ARG=""
+# When project is not ml-pipeline-test, VMs need permission to fetch some images in gcr.io/ml-pipeline-test.
+if [ "$PROJECT" != "ml-pipeline-test" ]; then
+  COPIED_IMAGE_BUILDER_IMAGE=${GCR_IMAGE_BASE_DIR}/image-builder
+  echo "Copy image builder image to ${COPIED_IMAGE_BUILDER_IMAGE}"
+  yes | gcloud container images add-tag \
+    gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 \
+    ${COPIED_IMAGE_BUILDER_IMAGE}:latest
+  IMAGE_BUILDER_ARG="-p image-builder-image=${COPIED_IMAGE_BUILDER_IMAGE}"
+fi
+
 # Build Images
 echo "submitting argo workflow to build docker images for commit ${PULL_PULL_SHA}..."
 ARGO_WORKFLOW=`argo submit ${DIR}/build_image.yaml \
 -p image-build-context-gcs-uri="$remote_code_archive_uri" \
+${IMAGE_BUILDER_ARG} \
 -p api-image="${GCR_IMAGE_BASE_DIR}/api-server" \
 -p frontend-image="${GCR_IMAGE_BASE_DIR}/frontend" \
 -p scheduledworkflow-image="${GCR_IMAGE_BASE_DIR}/scheduledworkflow" \
@@ -95,6 +111,7 @@ source ${DIR}/deploy-pipeline.sh --gcr_image_base_dir ${GCR_IMAGE_BASE_DIR}
 echo "submitting argo workflow to run tests for commit ${PULL_PULL_SHA}..."
 ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
 -p image-build-context-gcs-uri="$remote_code_archive_uri" \
+${IMAGE_BUILDER_ARG} \
 -p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
 -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
 -p cluster-type="${CLUSTER_TYPE}" \

test/sample_test.yaml

@@ -25,6 +25,8 @@ spec:
   arguments:
     parameters:
       - name: image-build-context-gcs-uri
+      - name: image-builder-image
+        value: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
       - name: target-image-prefix
       - name: test-results-gcs-dir
       - name: sample-tests-image-suffix
@@ -90,7 +92,7 @@ spec:
             valueFrom:
               path: /outputs/strict-image-name/file
       container:
-        image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
+        image: "{{workflow.parameters.image-builder-image}}"
         imagePullPolicy: 'Always'
         args: [
           "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",

test/test-prep.sh

@@ -16,8 +16,10 @@
 
 set -x
 
-# activating the service account
-gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+  # activating the service account
+  gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+fi
 gcloud config set compute/zone us-east1-b
 gcloud config set core/project ${PROJECT}