Changes to support running E2E tests on minikube. (#523)

*  Changes to support running E2E tests on minikube.

 * Parameterize the E2E workflow with a parameter that controls the platform
   we are running on.

* Related to kubeflow/testing#6

* Add environment variable to configure KUBECONFIG.

* Split test_deploy into separate functions to deploy the kubeflow APP
      and get GKE credentials

* This allows us to handle credentials and the kubeconfig file consistently
      for GKE and minikube; on subsequent steps we just set KUBECONFIG to
      point to the config.

* Refactor the workflow to be a DAG.

* Update the TFServing workflow to use the new version of test_deploy
      which uses a separate step to get credentials and store them in .kube_config.

* The name for the test_runner step needs to be unique so that we will
      create different test cases in gubernator for the different platforms.

* Parameterize the E2E workflow by a platform parameter that controls whether we use minikube or gke.

* Always store kube config on an NFS share.

* Fix lint and add a style .yapf file to autoformat files copied for tf-operator.

* Fix merge issue.

* Autoformat jsonnet file.

* * Address reviewer comments.

* Fix bad merge in workflow.libsonnet.

.style.yapf

@@ -0,0 +1,4 @@
+[style]
+based_on_style = google
+indent_width: 2
+continuation_indent_width: 2

components/k8s-model-server/images/releaser/components/workflows.libsonnet

@@ -134,6 +134,12 @@
                 },
               },
             },
+            // We use a directory in our NFS share to store our kube config. 
+            // This way we can configure it on a single step and reuse it on subsequent steps.
+            {
+              name: "KUBECONFIG",
+              value: testDir + "/.kube/config",
+            },
           ] + prow_env + env_vars,
           volumeMounts: [
             {
@@ -220,13 +226,17 @@
           name: "checkout",
           template: "checkout",
         },
-
         {
           name: "create-pr-symlink",
           template: "create-pr-symlink",
           dependencies: ["checkout"],
         },
 
+        {
+          name: "setup",
+          template: "setup",
+          dependencies: ["checkout"],
+        },
         {
           name: "test-tf-serving",
           template: "test-tf-serving",
@@ -241,7 +251,7 @@
         // {
         //   name: "test-tf-serving-gpu",
         //   template: "test-tf-serving-gpu",
-        //   dependencies: ["deploy-tf-serving-gpu"],
+        //   dependencies: ["deploy-tf-serving-gpu",],
         // },
       ];
       local e2e_tasks = e2e_tasks_base + if build_image then [
@@ -253,22 +263,20 @@
         {
           name: "deploy-tf-serving",
           template: "deploy-tf-serving",
-          dependencies: ["build-tf-serving-cpu"],
+          dependencies: ["build-tf-serving-cpu", "setup"],
         },
       ] else [
         {
           name: "deploy-tf-serving",
           template: "deploy-tf-serving",
-          dependencies: ["checkout"],
+          dependencies: ["setup"],
         },
       ];
       local deploy_tf_serving_command_base = [
         "python",
         "-m",
         "testing.test_deploy",
-        "--project=" + project,
-        "--cluster=" + cluster,
-        "--zone=" + zone,
+        "--project=" + project,        
         "--github_token=$(GITHUB_TOKEN)",
         // TODO(jlewi): This is duplicative with params. We should probably get
         // rid of this and just treat namespace as another parameter.
@@ -355,6 +363,19 @@
 
             buildImageTemplate("build-tf-serving-cpu", "Dockerfile.cpu", cpuImage),
 
+            // Setup configures a kubeconfig file for GKE.
+            buildTemplate("setup", [
+              "python",
+              "-m",
+              "testing.test_deploy",
+              "--project=" + project,
+              "--test_dir=" + testDir,
+              "--artifacts_dir=" + artifactsDir,
+              "get_gke_credentials",
+              "--cluster=" + cluster,
+              "--zone=" + zone,
+            ]),  // setup
+
             buildTemplate(
               "deploy-tf-serving",
               deploy_tf_serving_command,
@@ -395,9 +416,7 @@
                 "-m",
                 "testing.test_deploy",
                 "--project=" + project,
-                "--cluster=" + cluster,
                 "--namespace=" + stepsNamespace,
-                "--zone=" + zone,
                 "--test_dir=" + testDir,
                 "--artifacts_dir=" + artifactsDir,
                 "teardown",

prow_config.yaml

@@ -1,9 +1,18 @@
 # This file configures the workflows to trigger in our Prow jobs.
 # see kubeflow/testing/py/run_e2e_workflow.py
 workflows:
+  # Run tests on GKE
   - app_dir: kubeflow/kubeflow/testing/workflows
     component: workflows
-    name: kubeflow-e2e
+    name: kubeflow-e2e-gke
+    params:
+      platform: gke
+  # Run tests on minikube
+  - app_dir: kubeflow/kubeflow/testing/workflows
+    component: workflows
+    name: kubeflow-e2e-minikube
+    params:
+      platform: minikube
   - app_dir: kubeflow/kubeflow/components/k8s-model-server/images/releaser
     component: workflows
     name: tf-serving-image

testing/install_minikube.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# A helper script to run on a VM to install minikube.
+# A helper script to run on a VM to install and start minikube.
 
 set -ex
 
@@ -32,5 +32,9 @@ curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.0/minik
 chmod +x minikube 
 sudo mv minikube /usr/local/bin/
 
+# We need a large disk for JupyterHub.
+sudo minikube start --vm-driver=none --disk-size=40g
+
+# Change the permissions because we will copy these files.
 sudo chmod -R a+rw ~/.kube
 sudo chmod -R a+rw ~/.minikube