fixes 'refactor tf-job-operator to match style-guide of libsonnet' (#…

…1535) * /retest * snapshot * snapshot * snapshot * all resources converted * consolidate rules * formatting, comments * remove test params, env * fixed tf-job-operator test * /retest * /retest * /retest * add scope to crd's and include tests * /retest * /retest * /retest * /retest * bump gke version * snapshot * for params.deploymentScope:cluster the crd should not have any scope which defaults to cluster and also passes validation * /retest
kubeflow · Sep 25, 2018 · 403d653 · 403d653
1 parent a6a2a12
commit 403d653
Show file tree

Hide file tree

Showing 3 changed files with 631 additions and 436 deletions.
diff --git a/kubeflow/core/prototypes/tf-job-operator.jsonnet b/kubeflow/core/prototypes/tf-job-operator.jsonnet
@@ -12,13 +12,6 @@
 // @optionalParam deploymentScope string cluster The scope at which tf-job-operator should be deployed - valid values are cluster, namespace.
 // @optionalParam deploymentNamespace string null The namespace to which tf-job-operator should be scoped. If deploymentScope is set to cluster, this is ignored.
 
-local k = import "k.libsonnet";
-local tfjob = import "kubeflow/core/tf-job-operator.libsonnet";
-
-// updatedParams uses the environment namespace if
-// the namespace parameter is not explicitly set
-local updatedParams = params {
-  namespace: if params.namespace == "null" then env.namespace else params.namespace,
-};
-
-std.prune(k.core.v1.list.new(tfjob.all(updatedParams)))
+local tfJobOperator = import "kubeflow/core/tf-job-operator.libsonnet";
+local instance = tfJobOperator.new(env, params);
+instance.list(instance.all)
diff --git a/kubeflow/core/tests/tf-job_test.jsonnet b/kubeflow/core/tests/tf-job_test.jsonnet
@@ -1,12 +1,29 @@
 local tfjob = import "../tf-job-operator.libsonnet";
-local params = {
-  namespace:: "test-kf-001",
+local paramsv1alpha1 = {
+  name:: "tf-job-operator",
+  tfJobImage:: "gcr.io/kubeflow-images-public/tf_operator:v20180226-403",
+  tfDefaultImage:: "null",
+  deploymentScope:: "cluster",
+  deploymentNamespace:: "null",
+  tfJobVersion: "v1alpha1",
+};
+local paramsv1alpha2 = {
+  name:: "tf-job-operator",
   tfJobImage:: "gcr.io/kubeflow-images-public/tf_operator:v20180226-403",
   tfDefaultImage:: "null",
+  deploymentScope:: "cluster",
+  deploymentNamespace:: "null",
+  tfJobVersion: "v1alpha2",
+};
+local env = {
+  namespace:: "test-kf-001",
 };
 
+local tfjobv1alpha1 = tfjob.new(env, paramsv1alpha1);
+local tfjobv1alpha2 = tfjob.new(env, paramsv1alpha2);
+
 std.assertEqual(
-  tfjob.parts(params.namespace).tfJobDeploy(params.tfJobImage),
+  tfjobv1alpha1.tfJobDeployment,
   {
     apiVersion: "extensions/v1beta1",
     kind: "Deployment",
@@ -75,7 +92,7 @@ std.assertEqual(
 ) &&
 
 std.assertEqual(
-  tfjob.parts(params.namespace).configMap(params.tfDefaultImage),
+  tfjobv1alpha1.tfConfigMap,
   {
     apiVersion: "v1",
     data: {
@@ -90,7 +107,7 @@ std.assertEqual(
 ) &&
 
 std.assertEqual(
-  tfjob.parts(params.namespace).serviceAccount,
+  tfjobv1alpha1.tfServiceAccount,
   {
     apiVersion: "v1",
     kind: "ServiceAccount",
@@ -105,7 +122,7 @@ std.assertEqual(
 ) &&
 
 std.assertEqual(
-  tfjob.parts(params.namespace).operatorRole("cluster", "default"),
+  tfjobv1alpha1.tfOperatorRole,
   {
     apiVersion: "rbac.authorization.k8s.io/v1beta1",
     kind: "ClusterRole",
@@ -194,7 +211,7 @@ std.assertEqual(
 ) &&
 
 std.assertEqual(
-  tfjob.parts(params.namespace).operatorRoleBinding("cluster", "default"),
+  tfjobv1alpha1.tfOperatorRoleBinding,
   {
     apiVersion: "rbac.authorization.k8s.io/v1beta1",
     kind: "ClusterRoleBinding",
@@ -220,7 +237,7 @@ std.assertEqual(
 ) &&
 
 std.assertEqual(
-  tfjob.parts(params.namespace).crd,
+  tfjobv1alpha1.tfJobCrd,
   {
     apiVersion: "apiextensions.k8s.io/v1beta1",
     kind: "CustomResourceDefinition",
@@ -237,4 +254,270 @@ std.assertEqual(
       version: "v1alpha1",
     },
   }
+) &&
+
+std.assertEqual(
+  tfjobv1alpha2.tfJobCrd,
+  {
+    apiVersion: "apiextensions.k8s.io/v1beta1",
+    kind: "CustomResourceDefinition",
+    metadata: {
+      name: "tfjobs.kubeflow.org",
+    },
+    spec: {
+      group: "kubeflow.org",
+      names: {
+        kind: "TFJob",
+        plural: "tfjobs",
+        singular: "tfjob",
+      },
+      validation: {
+        openAPIV3Schema: {
+          properties: {
+            spec: {
+              properties: {
+                tfReplicaSpecs: {
+                  properties: {
+                    Chief: {
+                      properties: {
+                        replicas: {
+                          maximum: 1,
+                          minimum: 1,
+                          type: "integer",
+                        },
+                      },
+                    },
+                    PS: {
+                      properties: {
+                        replicas: {
+                          minimum: 1,
+                          type: "integer",
+                        },
+                      },
+                    },
+                    Worker: {
+                      properties: {
+                        replicas: {
+                          minimum: 1,
+                          type: "integer",
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      version: "v1alpha2",
+    },
+  }
+) &&
+
+std.assertEqual(
+  tfjobv1alpha2.tfJobDeployment,
+  {
+    apiVersion: "extensions/v1beta1",
+    kind: "Deployment",
+    metadata: {
+      name: "tf-job-operator-v1alpha2",
+      namespace: "test-kf-001",
+    },
+    spec: {
+      replicas: 1,
+      template: {
+        metadata: {
+          labels: {
+            name: "tf-job-operator",
+          },
+        },
+        spec: {
+          containers: [
+            {
+              command: [
+                "/opt/kubeflow/tf-operator.v2",
+                "--alsologtostderr",
+                "-v=1",
+              ],
+              env: [
+                {
+                  name: "MY_POD_NAMESPACE",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "metadata.namespace",
+                    },
+                  },
+                },
+                {
+                  name: "MY_POD_NAME",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "metadata.name",
+                    },
+                  },
+                },
+              ],
+              image: "gcr.io/kubeflow-images-public/tf_operator:v20180226-403",
+              name: "tf-job-operator",
+              volumeMounts: [
+                {
+                  mountPath: "/etc/config",
+                  name: "config-volume",
+                },
+              ],
+            },
+          ],
+          serviceAccountName: "tf-job-operator",
+          volumes: [
+            {
+              configMap: {
+                name: "tf-job-operator-config",
+              },
+              name: "config-volume",
+            },
+          ],
+        },
+      },
+    },
+  }
+) &&
+
+std.assertEqual(
+  tfjobv1alpha2.tfUiDeployment,
+  {
+    apiVersion: "extensions/v1beta1",
+    kind: "Deployment",
+    metadata: {
+      name: "tf-job-dashboard",
+      namespace: "test-kf-001",
+    },
+    spec: {
+      template: {
+        metadata: {
+          labels: {
+            name: "tf-job-dashboard",
+          },
+        },
+        spec: {
+          containers: [
+            {
+              command: [
+                "/opt/tensorflow_k8s/dashboard/backend",
+              ],
+              env: [
+                {
+                  name: "KUBEFLOW_NAMESPACE",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "metadata.namespace",
+                    },
+                  },
+                },
+              ],
+              image: "gcr.io/kubeflow-images-public/tf_operator:v20180226-403",
+              name: "tf-job-dashboard",
+              ports: [
+                {
+                  containerPort: 8080,
+                },
+              ],
+            },
+          ],
+          serviceAccountName: "tf-job-dashboard",
+        },
+      },
+    },
+  }
+) &&
+
+std.assertEqual(
+  tfjobv1alpha2.tfUiRole,
+  {
+    apiVersion: "rbac.authorization.k8s.io/v1beta1",
+    kind: "ClusterRole",
+    metadata: {
+      labels: {
+        app: "tf-job-dashboard",
+      },
+      name: "tf-job-dashboard",
+    },
+    rules: [
+      {
+        apiGroups: [
+          "tensorflow.org",
+          "kubeflow.org",
+        ],
+        resources: [
+          "tfjobs",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+      {
+        apiGroups: [
+          "apiextensions.k8s.io",
+        ],
+        resources: [
+          "customresourcedefinitions",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+      {
+        apiGroups: [
+          "storage.k8s.io",
+        ],
+        resources: [
+          "storageclasses",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+      {
+        apiGroups: [
+          "batch",
+        ],
+        resources: [
+          "jobs",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+      {
+        apiGroups: [
+          "",
+        ],
+        resources: [
+          "configmaps",
+          "pods",
+          "services",
+          "endpoints",
+          "persistentvolumeclaims",
+          "events",
+          "pods/log",
+          "namespaces",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+      {
+        apiGroups: [
+          "apps",
+          "extensions",
+        ],
+        resources: [
+          "deployments",
+        ],
+        verbs: [
+          "*",
+        ],
+      },
+    ],
+  }
 )