fixes 'Move components in core into a GCP specific ksonnet package' (#…

…2043)

* fixes 'Move components in core into a GCP specific ksonnet package'

* remove gcp related packages from core

* /retest

* update approvers and reviewers

* remove gaocegege

kubeflow/gcp/OWNERS

@@ -0,0 +1,7 @@
+approvers:
+  - abhi-g
+  - jlewi
+  - kunming
+  - llunnn
+  - r2d4
+  - richardsliu

kubeflow/gcp/README.md

@@ -0,0 +1,11 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [core](#core)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# gcp
+
+This ksonnet package contains GCP specific prototypes. 

kubeflow/core/cert-manager.libsonnet → kubeflow/gcp/cert-manager.libsonnet

@@ -1,5 +1,5 @@
 {
-  local util = import "kubeflow/core/util.libsonnet",
+  local k = import "k.libsonnet",
   new(_env, _params):: {
     local params = _params + _env,
 
@@ -185,6 +185,6 @@
       self.issuerLEProd,
     ],
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/core/cloud-endpoints.libsonnet → kubeflow/gcp/cloud-endpoints.libsonnet

@@ -1,6 +1,5 @@
 {
   local k = import "k.libsonnet",
-  local util = import "kubeflow/core/util.libsonnet",
   new(_env, _params):: {
     local params = _params + _env {
       cloudEndpointsImage: "gcr.io/cloud-solutions-group/cloud-endpoints-controller:0.1.1",
@@ -360,6 +359,6 @@
       Services +
       Deployments,
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/core/google-cloud-filestore-pv.libsonnet → kubeflow/gcp/google-cloud-filestore-pv.libsonnet

@@ -1,6 +1,5 @@
 {
   local k = import "k.libsonnet",
-  local util = import "kubeflow/core/util.libsonnet",
   new(_env, _params):: {
     local params = _params + _env,
 
@@ -99,6 +98,6 @@
       self.gcfsPersmissions,
     ],
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/core/iap.libsonnet → kubeflow/gcp/iap.libsonnet

@@ -1004,6 +1004,6 @@
       self.cloudEndpoint,
     ],
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/core/metric-collector.libsonnet → kubeflow/gcp/metric-collector.libsonnet

@@ -1,6 +1,5 @@
 {
   local k = import "k.libsonnet",
-  local util = import "kubeflow/core/util.libsonnet",
   new(_env, _params):: {
     local params = _params + _env,
 
@@ -185,6 +184,6 @@
       self.deploy,
     ],
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/gcp/parts.yaml

@@ -0,0 +1,35 @@
+{
+   "name": "gcp",
+   "apiVersion": "0.0.1",
+   "kind": "ksonnet.io/parts",
+   "description": "Core components of Kubeflow.\n",
+   "author": "kubeflow team <kubeflow-team@google.com>",
+   "contributors": [
+      {
+         "name": "Jeremy Lewi",
+         "email": "jlewi@google.com"
+      }
+   ],
+   "repository": {
+      "type": "git",
+      "url": "https://github.com/kubeflow/kubeflow"
+   },
+   "bugs": {
+      "url": "https://github.com/kubeflow/kubeflow/issues"
+   },
+   "keywords": [
+      "kubeflow",
+      "tensorflow"
+   ],
+   "quickStart": {
+      "prototype": "io.ksonnet.pkg.kubeflow",
+      "componentName": "gcp",
+      "flags": {
+         "name": "gcp",
+         "namespace": "default",
+         "disks": ""
+      },
+      "comment": "GCP specific Kubeflow components."
+   },
+   "license": "Apache 2.0"
+}

kubeflow/core/prometheus.libsonnet → kubeflow/gcp/prometheus.libsonnet

@@ -1,6 +1,5 @@
 {
   local k = import "k.libsonnet",
-  local util = import "kubeflow/core/util.libsonnet",
   new(_env, _params):: {
     local params = _params + _env,
 
@@ -217,6 +216,6 @@
       self.deployment,
     ],
 
-    list(obj=self.all):: util.list(obj),
+    list(obj=self.all):: k.core.v1.list.new(obj,),
   },
 }

kubeflow/core/prototypes/cert-manager.jsonnet → kubeflow/gcp/prototypes/cert-manager.jsonnet

@@ -7,6 +7,6 @@
 // @optionalParam acmeUrl string https://acme-v02.api.letsencrypt.org/directory The ACME server URL, set to https://acme-staging-v02.api.letsencrypt.org/directory for staging API.
 // @optionalParam certManagerImage string quay.io/jetstack/cert-manager-controller:v0.4.0 certManagerImage
 
-local certManager = import "kubeflow/core/cert-manager.libsonnet";
+local certManager = import "kubeflow/gcp/cert-manager.libsonnet";
 local instance = certManager.new(env, params);
 instance.list(instance.all)

kubeflow/core/prototypes/cloud-endpoints.jsonnet → kubeflow/gcp/prototypes/cloud-endpoints.jsonnet

@@ -6,6 +6,6 @@
 // @optionalParam secretName string admin-gcp-sa Name of secret containing the json service account key.
 // @optionalParam secretKey string admin-gcp-sa.json Name of the key in the secret containing the JSON service account key.
 
-local cloudEndpoints = import "kubeflow/core/cloud-endpoints.libsonnet";
+local cloudEndpoints = import "kubeflow/gcp/cloud-endpoints.libsonnet";
 local instance = cloudEndpoints.new(env, params);
 instance.list(instance.all)

kubeflow/core/prototypes/google-cloud-filestore-pv.jsonnet → kubeflow/gcp/prototypes/google-cloud-filestore-pv.jsonnet

@@ -8,6 +8,6 @@
 // @param serverIP string Google Cloud Filestore Server IP
 // @optionalParam image string gcr.io/kubeflow-images-public/ubuntu:18.04 The docker image to use
 
-local google_cloud_file_store_pv = import "kubeflow/core/google-cloud-filestore-pv.libsonnet";
+local google_cloud_file_store_pv = import "kubeflow/gcp/google-cloud-filestore-pv.libsonnet";
 local instance = google_cloud_file_store_pv.new(env, params);
 instance.list(instance.all)

kubeflow/core/prototypes/iap-ingress.jsonnet → kubeflow/gcp/prototypes/iap-ingress.jsonnet

@@ -13,6 +13,6 @@
 // @optionalParam oauthSecretName string kubeflow-oauth The name of the secret containing the OAuth client_id and client_secret.
 // @optionalParam privateGKECluster string false Is the k8s cluster a private GKE cluster
 
-local iap = import "kubeflow/core/iap.libsonnet";
+local iap = import "kubeflow/gcp/iap.libsonnet";
 local instance = iap.new(env, params);
 instance.list(instance.all)

kubeflow/core/prototypes/metric-collector.jsonnet → kubeflow/gcp/prototypes/metric-collector.jsonnet

@@ -8,6 +8,6 @@
 // @optionalParam metricImage string gcr.io/kubeflow-images-public/metric-collector:latest Image for running metric exporter of kubeflow availability.
 // @optionalParam oauthSecretName string kubeflow-oauth The name of the secret containing the OAuth client_id and client_secret.
 
-local metricCollector = import "kubeflow/core/metric-collector.libsonnet";
+local metricCollector = import "kubeflow/gcp/metric-collector.libsonnet";
 local instance = metricCollector.new(env, params);
 instance.list(instance.all)

kubeflow/core/prototypes/prometheus.jsonnet → kubeflow/gcp/prototypes/prometheus.jsonnet

@@ -1,12 +1,12 @@
 // @apiVersion 0.1
 // @name io.ksonnet.pkg.prometheus
-// @description Provides prometheus prototype in kubeflow core.
+// @description Provides prometheus prototype in kubeflow gcp package
 // @shortDescription Prometheus Service.
 // @param name string Name for the component
 // @param projectId string GCP project id.
 // @param clusterName string GKE cluster name.
 // @param zone string GKE cluster zone.
 
-local prometheus = import "kubeflow/core/prometheus.libsonnet";
+local prometheus = import "kubeflow/gcp/prometheus.libsonnet";
 local instance = prometheus.new(env, params);
 instance.list(instance.all)

scripts/gke/util.sh

@@ -223,6 +223,10 @@ gcpGenerateKsApp() {
   pushd .
   cd "${KUBEFLOW_KS_DIR}"
 
+  # Install the gcp package 
+  ks pkg install kubeflow/gcp
+
+  # Generate all required components
   ks generate cloud-endpoints cloud-endpoints
   ks generate cert-manager cert-manager --acmeEmail=${EMAIL}
   ks generate iap-ingress iap-ingress --ipName=${KUBEFLOW_IP_NAME} --hostname=${KUBEFLOW_HOSTNAME}

testing/deploy_utils.py

@@ -102,7 +102,7 @@ def setup_kubeflow_ks_app(dir, namespace, github_token, api_client):
     ["ks", "registry", "add", "kubeflow", kubeflow_registry], cwd=app_dir)
 
   # Install required packages
-  packages = ["kubeflow/core", "kubeflow/jupyter", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/tf-training", "kubeflow/pytorch-job", "kubeflow/argo"]
+  packages = ["kubeflow/core", "kubeflow/gcp", "kubeflow/jupyter", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/tf-training", "kubeflow/pytorch-job", "kubeflow/argo"]
 
   # Instead of installing packages we edit the app.yaml file directly
   #for p in packages:

testing/test_jsonnet.py

@@ -22,7 +22,7 @@
 
 Example invocation
 
-python python -m testing.test_jsonnet --test_files_dirs=/kubeflow/application/tests,/kubeflow/core/tests,/kubeflow/jupyter/tests,/kubeflow/iap/tests,/kubeflow/tensorboard/tests,/kubeflow/examples/tests,/kubeflow/metacontroller/tests,/kubeflow/profiles/tests,/kubeflow/tf-training/tests --artifacts_dir=/tmp/artifacts
+python python -m testing.test_jsonnet --test_files_dirs=/kubeflow/application/tests,/kubeflow/core/tests,/kubeflow/jupyter/tests,/kubeflow/iap/tests,/kubeflow/gcp/tests,/kubeflow/tensorboard/tests,/kubeflow/examples/tests,/kubeflow/metacontroller/tests,/kubeflow/profiles/tests,/kubeflow/tf-training/tests --artifacts_dir=/tmp/artifacts
 
 """