Add more tests to the subgraph we created to run the tests. (#1342)

* Add more tests to the subgraph we created to run the tests.

* Update wait_for_deployment.py to use KUBECONFIG so its not limited to GKE.

* I don't think we actually need wait_for_deployment; all it does is wait
  for the CRD to be created and that's not very useful. It would
  be better to wait for the actual controller deployments to start.

* Create a script wait_for_kubeflow based on deploy_kubeflow.py that
  waits for Kubeflow to be deployed and performs basic checks like
  ensuring everything started correctly.

* Fix a typo in wait_for_kubeflow
* Add steps to copy artifacts to prow bucket.
* Create a workflow for unittests."

* Don't run workflows.jsonnet on presubmit; we use kfctl_test now.

* Add an option to kfctl_test to not delete the cluster; useful
  for leaving it up to debug tests.

* latest.

* Autoformat.

prow_config.yaml

@@ -1,16 +1,7 @@
 # This file configures the workflows to trigger in our Prow jobs.
 # see kubeflow/testing/py/run_e2e_workflow.py
 workflows:
-  # Run tests on GKE
-  - app_dir: kubeflow/kubeflow/testing/workflows
-    component: workflows
-    name: kubeflow-e2e-gke
-    job_types:
-      - presubmit
-      - postsubmit
-    params:
-      platform: gke
-  # kfctl test
+  # kfctl test runs tests on gke.
   - app_dir: kubeflow/kubeflow/testing/workflows
     component: kfctl_test
     name: kfctl
@@ -23,6 +14,19 @@ workflows:
       - testing/*
     params:
       platform: gke
+  # Run unittests
+  # TODO(jlewi): Need to add step to run go and python unittests
+  - app_dir: kubeflow/kubeflow/testing/workflows
+    component: unit_tests
+    name: unittests    
+  # TODO(jlewi): We should be running the minikube workflow
+  # on presubmit when the minikube E2E test itself changes
+  # so we verify the test is working before submitting
+  # changes to it. But right now we can't match a regex or
+  # glob.
+  # see: https://github.com/kubeflow/testing/issues/187
+  # see: https://github.com/kubeflow/kubeflow/issues/1350
+  #   
   # Run tests on minikube
   - app_dir: kubeflow/kubeflow/testing/workflows
     component: workflows

testing/deploy_kubeflow.py

@@ -1,3 +1,8 @@
+"""Deploy Kubeflow and wait for it to be deployed.
+
+TODO(jlewi): This script is outdated. Its no longer used for GKE.
+It is still used by minikube. For minikube we should be using kfctl.sh
+"""
 import argparse
 import logging
 import os

testing/wait_for_deployment.py

@@ -20,6 +20,11 @@
 
 python -m testing.wait_for_deployment --cluster=kubeflow-testing --project=kubeflow-ci --zone=us-east1-d --timeout=3
 
+
+TODO(jlewi): Waiting for the CRD's to be created probably isn't that useful.
+I think that will be nearly instantaneous. If we're going to wait for something
+it should probably be waiting for the controllers to actually be deployed.
+We can probably get rid of this and just use wait_for_kubeflow.py.
 """
 
 from __future__ import print_function
@@ -36,21 +41,6 @@
 
 def parse_args():
   parser = argparse.ArgumentParser()
-  parser.add_argument(
-    "--cluster",
-    default="",
-    type=str,
-    help="Cluster Name")
-  parser.add_argument(
-    "--zone",
-    default="",
-    type=str,
-    help="Zone Name")
-  parser.add_argument(
-    "--project",
-    default="",
-    type=str,
-    help="Project ID")
   parser.add_argument(
     "--timeout",
     default=5,
@@ -74,7 +64,7 @@ def wait_for_resource(resource, end_time):
 def test_wait_for_deployment(test_case): # pylint: disable=redefined-outer-name
   args = parse_args()
   util.maybe_activate_service_account()
-  util.run(["gcloud",  "container", "clusters", "get-credentials", args.cluster, "--zone=" + args.zone,  "--project=" + args.project])
+  util.load_kube_config()
   end_time =  datetime.datetime.now() + datetime.timedelta(0, args.timeout*60)
   wait_for_resource("crd/tfjobs.kubeflow.org", end_time)
   wait_for_resource("crd/pytorchjobs.kubeflow.org", end_time)

testing/wait_for_kubeflow.py

@@ -0,0 +1,47 @@
+"""Wait for Kubeflow to be deployed."""
+import argparse
+import logging
+
+from testing import deploy_utils
+from kubeflow.testing import test_helper
+from kubeflow.testing import util  # pylint: disable=no-name-in-module
+
+def parse_args():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    "--namespace", default=None, type=str, help=("The namespace to use."))
+
+  args, _ = parser.parse_known_args()
+  return args
+
+def deploy_kubeflow(_):
+  """Deploy Kubeflow."""
+  args = parse_args()
+  namespace = args.namespace
+  api_client = deploy_utils.create_k8s_client()
+
+  util.load_kube_config()
+  # Verify that the TfJob operator is actually deployed.
+  tf_job_deployment_name = "tf-job-operator-v1alpha2"
+  logging.info("Verifying TfJob controller started.")
+  util.wait_for_deployment(api_client, namespace, tf_job_deployment_name)
+
+  # Verify that JupyterHub is actually deployed.
+  jupyterhub_name = "tf-hub"
+  logging.info("Verifying TfHub started.")
+  util.wait_for_statefulset(api_client, namespace, jupyterhub_name)
+
+  # Verify that PyTorch Operator actually deployed
+  pytorch_operator_deployment_name = "pytorch-operator"
+  logging.info("Verifying PyTorchJob controller started.")
+  util.wait_for_deployment(api_client, namespace, pytorch_operator_deployment_name)
+
+def main():
+  test_case = test_helper.TestCase(
+    name='deploy_kubeflow', test_func=deploy_kubeflow)
+  test_suite = test_helper.init(
+    name='deploy_kubeflow', test_cases=[test_case])
+  test_suite.run()
+
+if __name__ == "__main__":
+  main()

testing/workflows/components/kfctl_test.jsonnet

@@ -133,7 +133,18 @@ local dagTemplates = [
                               value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD",
                             }]),
     dependencies: null,
-  },
+  },  // checkout
+  {
+    template: buildTemplate("create-pr-symlink", [
+      "python",
+      "-m",
+      "kubeflow.testing.prow_artifacts",
+      "--artifacts_dir=" + outputDir,
+      "create_pr_symlink",
+      "--bucket=" + bucket,
+    ]),  // create-pr-symlink
+    dependencies: ["checkout"],
+  },  // create-pr-symlink
   {
     template: buildTemplate(
       "kfctl-init",
@@ -225,8 +236,10 @@ local dagTemplates = [
 
 // Each item is a dictionary describing one step in the graph
 // to execute on exit
-local exitTemplates = [
-  {
+local deleteKubeflow = util.toBool(params.deleteKubeflow);
+
+local deleteStep = if deleteKubeflow then
+  [{
     template: buildTemplate(
       "kfctl-delete",
       [
@@ -238,23 +251,41 @@ local exitTemplates = [
       working_dir=appDir
     ),
     dependencies: null,
-  },
-  {
-    template:
-      buildTemplate("test-dir-delete", [
+  }]
+else [];
+
+local exitTemplates =
+  deleteStep +
+  [
+    {
+      template: buildTemplate("copy-artifacts", [
         "python",
         "-m",
-        "testing.run_with_retry",
-        "--retries=5",
-        "--",
-        "rm",
-        "-rf",
-        testDir,
-      ]),  // test-dir-delete
-    dependencies: ["kfctl-delete"],
-  },
+        "kubeflow.testing.prow_artifacts",
+        "--artifacts_dir=" + outputDir,
+        "copy_artifacts",
+        "--bucket=" + bucket,
+      ]),  // copy-artifacts,
 
-];
+      dependencies: if deleteKubeflow then
+        ["kfctl-delete"]
+      else null,
+    },
+    {
+      template:
+        buildTemplate("test-dir-delete", [
+          "python",
+          "-m",
+          "testing.run_with_retry",
+          "--retries=5",
+          "--",
+          "rm",
+          "-rf",
+          testDir,
+        ]),  // test-dir-delete
+      dependencies: ["copy-artifacts"],
+    },
+  ];
 
 // Dag defines the tasks in the graph
 local dag = {

testing/workflows/components/params.libsonnet

@@ -27,6 +27,13 @@
       name: "somefakename",
       namespace: "kubeflow-test-infra",
       prow_env: "",
+      deleteKubeflow: true,
+    },
+    unit_tests: {
+      bucket: "kubeflow-ci_temp",
+      name: "somefakename",
+      namespace: "kubeflow-test-infra",
+      prow_env: "",
     },
   },
 }