From d758175b7c42587aa6efe65c39d4a9361bc1b9a9 Mon Sep 17 00:00:00 2001 From: Kam Kasravi Date: Mon, 10 Dec 2018 11:52:21 -0800 Subject: [PATCH] fixes 'Move components in core into a GCP specific ksonnet package' (#2043) * fixes 'Move components in core into a GCP specific ksonnet package' * remove gcp related packages from core * /retest * update approvers and reviewers * remove gaocegege --- kubeflow/gcp/OWNERS | 7 ++++ kubeflow/gcp/README.md | 11 ++++++ kubeflow/{core => gcp}/cert-manager.libsonnet | 4 +-- .../{core => gcp}/cloud-endpoints.libsonnet | 3 +- .../{core => gcp}/configure_envoy_for_iap.sh | 0 .../google-cloud-filestore-pv.libsonnet | 3 +- kubeflow/{core => gcp}/iap.libsonnet | 2 +- kubeflow/{core => gcp}/ingress_bootstrap.sh | 0 .../{core => gcp}/metric-collector.libsonnet | 3 +- kubeflow/gcp/parts.yaml | 35 +++++++++++++++++++ kubeflow/{core => gcp}/prometheus.libsonnet | 3 +- kubeflow/{core => gcp}/prometheus.yml | 0 .../prototypes/cert-manager.jsonnet | 2 +- .../prototypes/cloud-endpoints.jsonnet | 2 +- .../google-cloud-filestore-pv.jsonnet | 2 +- .../prototypes/iap-ingress.jsonnet | 2 +- .../prototypes/metric-collector.jsonnet | 2 +- .../prototypes/prometheus.jsonnet | 4 +-- kubeflow/{core => gcp}/setup_backend.sh | 0 kubeflow/{core => gcp}/tests/iap_test.jsonnet | 0 kubeflow/{core => gcp}/update_backend.sh | 0 scripts/gke/util.sh | 4 +++ testing/deploy_utils.py | 2 +- testing/test_jsonnet.py | 2 +- 24 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 kubeflow/gcp/OWNERS create mode 100644 kubeflow/gcp/README.md rename kubeflow/{core => gcp}/cert-manager.libsonnet (97%) rename kubeflow/{core => gcp}/cloud-endpoints.libsonnet (99%) rename kubeflow/{core => gcp}/configure_envoy_for_iap.sh (100%) rename kubeflow/{core => gcp}/google-cloud-filestore-pv.libsonnet (96%) rename kubeflow/{core => gcp}/iap.libsonnet (99%) rename kubeflow/{core => gcp}/ingress_bootstrap.sh (100%) rename kubeflow/{core => gcp}/metric-collector.libsonnet (97%) create mode 100644 kubeflow/gcp/parts.yaml rename kubeflow/{core => gcp}/prometheus.libsonnet (98%) rename kubeflow/{core => gcp}/prometheus.yml (100%) rename kubeflow/{core => gcp}/prototypes/cert-manager.jsonnet (90%) rename kubeflow/{core => gcp}/prototypes/cloud-endpoints.jsonnet (88%) rename kubeflow/{core => gcp}/prototypes/google-cloud-filestore-pv.jsonnet (86%) rename kubeflow/{core => gcp}/prototypes/iap-ingress.jsonnet (96%) rename kubeflow/{core => gcp}/prototypes/metric-collector.jsonnet (91%) rename kubeflow/{core => gcp}/prototypes/prometheus.jsonnet (72%) rename kubeflow/{core => gcp}/setup_backend.sh (100%) rename kubeflow/{core => gcp}/tests/iap_test.jsonnet (100%) rename kubeflow/{core => gcp}/update_backend.sh (100%) diff --git a/kubeflow/gcp/OWNERS b/kubeflow/gcp/OWNERS new file mode 100644 index 00000000000..e9769811898 --- /dev/null +++ b/kubeflow/gcp/OWNERS @@ -0,0 +1,7 @@ +approvers: + - abhi-g + - jlewi + - kunming + - llunnn + - r2d4 + - richardsliu diff --git a/kubeflow/gcp/README.md b/kubeflow/gcp/README.md new file mode 100644 index 00000000000..36f882878a8 --- /dev/null +++ b/kubeflow/gcp/README.md @@ -0,0 +1,11 @@ + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [core](#core) + + + +# gcp + +This ksonnet package contains GCP specific prototypes. diff --git a/kubeflow/core/cert-manager.libsonnet b/kubeflow/gcp/cert-manager.libsonnet similarity index 97% rename from kubeflow/core/cert-manager.libsonnet rename to kubeflow/gcp/cert-manager.libsonnet index b5a54bc6aa4..25853b34c5f 100644 --- a/kubeflow/core/cert-manager.libsonnet +++ b/kubeflow/gcp/cert-manager.libsonnet @@ -1,5 +1,5 @@ { - local util = import "kubeflow/core/util.libsonnet", + local k = import "k.libsonnet", new(_env, _params):: { local params = _params + _env, @@ -185,6 +185,6 @@ self.issuerLEProd, ], - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/core/cloud-endpoints.libsonnet b/kubeflow/gcp/cloud-endpoints.libsonnet similarity index 99% rename from kubeflow/core/cloud-endpoints.libsonnet rename to kubeflow/gcp/cloud-endpoints.libsonnet index a60ad7afeb7..18d20f118b1 100644 --- a/kubeflow/core/cloud-endpoints.libsonnet +++ b/kubeflow/gcp/cloud-endpoints.libsonnet @@ -1,6 +1,5 @@ { local k = import "k.libsonnet", - local util = import "kubeflow/core/util.libsonnet", new(_env, _params):: { local params = _params + _env { cloudEndpointsImage: "gcr.io/cloud-solutions-group/cloud-endpoints-controller:0.1.1", @@ -360,6 +359,6 @@ Services + Deployments, - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/core/configure_envoy_for_iap.sh b/kubeflow/gcp/configure_envoy_for_iap.sh similarity index 100% rename from kubeflow/core/configure_envoy_for_iap.sh rename to kubeflow/gcp/configure_envoy_for_iap.sh diff --git a/kubeflow/core/google-cloud-filestore-pv.libsonnet b/kubeflow/gcp/google-cloud-filestore-pv.libsonnet similarity index 96% rename from kubeflow/core/google-cloud-filestore-pv.libsonnet rename to kubeflow/gcp/google-cloud-filestore-pv.libsonnet index 708dc34f264..bec55b344ea 100644 --- a/kubeflow/core/google-cloud-filestore-pv.libsonnet +++ b/kubeflow/gcp/google-cloud-filestore-pv.libsonnet @@ -1,6 +1,5 @@ { local k = import "k.libsonnet", - local util = import "kubeflow/core/util.libsonnet", new(_env, _params):: { local params = _params + _env, @@ -99,6 +98,6 @@ self.gcfsPersmissions, ], - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/core/iap.libsonnet b/kubeflow/gcp/iap.libsonnet similarity index 99% rename from kubeflow/core/iap.libsonnet rename to kubeflow/gcp/iap.libsonnet index eca7c2cd4db..a1a42dc3953 100644 --- a/kubeflow/core/iap.libsonnet +++ b/kubeflow/gcp/iap.libsonnet @@ -1004,6 +1004,6 @@ self.cloudEndpoint, ], - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/core/ingress_bootstrap.sh b/kubeflow/gcp/ingress_bootstrap.sh similarity index 100% rename from kubeflow/core/ingress_bootstrap.sh rename to kubeflow/gcp/ingress_bootstrap.sh diff --git a/kubeflow/core/metric-collector.libsonnet b/kubeflow/gcp/metric-collector.libsonnet similarity index 97% rename from kubeflow/core/metric-collector.libsonnet rename to kubeflow/gcp/metric-collector.libsonnet index 47eb648474b..87fcc2ecf25 100644 --- a/kubeflow/core/metric-collector.libsonnet +++ b/kubeflow/gcp/metric-collector.libsonnet @@ -1,6 +1,5 @@ { local k = import "k.libsonnet", - local util = import "kubeflow/core/util.libsonnet", new(_env, _params):: { local params = _params + _env, @@ -185,6 +184,6 @@ self.deploy, ], - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/gcp/parts.yaml b/kubeflow/gcp/parts.yaml new file mode 100644 index 00000000000..e9f36029e0f --- /dev/null +++ b/kubeflow/gcp/parts.yaml @@ -0,0 +1,35 @@ +{ + "name": "gcp", + "apiVersion": "0.0.1", + "kind": "ksonnet.io/parts", + "description": "Core components of Kubeflow.\n", + "author": "kubeflow team ", + "contributors": [ + { + "name": "Jeremy Lewi", + "email": "jlewi@google.com" + } + ], + "repository": { + "type": "git", + "url": "https://github.com/kubeflow/kubeflow" + }, + "bugs": { + "url": "https://github.com/kubeflow/kubeflow/issues" + }, + "keywords": [ + "kubeflow", + "tensorflow" + ], + "quickStart": { + "prototype": "io.ksonnet.pkg.kubeflow", + "componentName": "gcp", + "flags": { + "name": "gcp", + "namespace": "default", + "disks": "" + }, + "comment": "GCP specific Kubeflow components." + }, + "license": "Apache 2.0" +} diff --git a/kubeflow/core/prometheus.libsonnet b/kubeflow/gcp/prometheus.libsonnet similarity index 98% rename from kubeflow/core/prometheus.libsonnet rename to kubeflow/gcp/prometheus.libsonnet index 78796101702..c5d1c86a747 100644 --- a/kubeflow/core/prometheus.libsonnet +++ b/kubeflow/gcp/prometheus.libsonnet @@ -1,6 +1,5 @@ { local k = import "k.libsonnet", - local util = import "kubeflow/core/util.libsonnet", new(_env, _params):: { local params = _params + _env, @@ -217,6 +216,6 @@ self.deployment, ], - list(obj=self.all):: util.list(obj), + list(obj=self.all):: k.core.v1.list.new(obj,), }, } diff --git a/kubeflow/core/prometheus.yml b/kubeflow/gcp/prometheus.yml similarity index 100% rename from kubeflow/core/prometheus.yml rename to kubeflow/gcp/prometheus.yml diff --git a/kubeflow/core/prototypes/cert-manager.jsonnet b/kubeflow/gcp/prototypes/cert-manager.jsonnet similarity index 90% rename from kubeflow/core/prototypes/cert-manager.jsonnet rename to kubeflow/gcp/prototypes/cert-manager.jsonnet index 023dc07808c..72a12cbe24c 100644 --- a/kubeflow/core/prototypes/cert-manager.jsonnet +++ b/kubeflow/gcp/prototypes/cert-manager.jsonnet @@ -7,6 +7,6 @@ // @optionalParam acmeUrl string https://acme-v02.api.letsencrypt.org/directory The ACME server URL, set to https://acme-staging-v02.api.letsencrypt.org/directory for staging API. // @optionalParam certManagerImage string quay.io/jetstack/cert-manager-controller:v0.4.0 certManagerImage -local certManager = import "kubeflow/core/cert-manager.libsonnet"; +local certManager = import "kubeflow/gcp/cert-manager.libsonnet"; local instance = certManager.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/prototypes/cloud-endpoints.jsonnet b/kubeflow/gcp/prototypes/cloud-endpoints.jsonnet similarity index 88% rename from kubeflow/core/prototypes/cloud-endpoints.jsonnet rename to kubeflow/gcp/prototypes/cloud-endpoints.jsonnet index 73ca899eb63..b5d5e950ccc 100644 --- a/kubeflow/core/prototypes/cloud-endpoints.jsonnet +++ b/kubeflow/gcp/prototypes/cloud-endpoints.jsonnet @@ -6,6 +6,6 @@ // @optionalParam secretName string admin-gcp-sa Name of secret containing the json service account key. // @optionalParam secretKey string admin-gcp-sa.json Name of the key in the secret containing the JSON service account key. -local cloudEndpoints = import "kubeflow/core/cloud-endpoints.libsonnet"; +local cloudEndpoints = import "kubeflow/gcp/cloud-endpoints.libsonnet"; local instance = cloudEndpoints.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/prototypes/google-cloud-filestore-pv.jsonnet b/kubeflow/gcp/prototypes/google-cloud-filestore-pv.jsonnet similarity index 86% rename from kubeflow/core/prototypes/google-cloud-filestore-pv.jsonnet rename to kubeflow/gcp/prototypes/google-cloud-filestore-pv.jsonnet index 0ffadbff0d5..0b41c765af1 100644 --- a/kubeflow/core/prototypes/google-cloud-filestore-pv.jsonnet +++ b/kubeflow/gcp/prototypes/google-cloud-filestore-pv.jsonnet @@ -8,6 +8,6 @@ // @param serverIP string Google Cloud Filestore Server IP // @optionalParam image string gcr.io/kubeflow-images-public/ubuntu:18.04 The docker image to use -local google_cloud_file_store_pv = import "kubeflow/core/google-cloud-filestore-pv.libsonnet"; +local google_cloud_file_store_pv = import "kubeflow/gcp/google-cloud-filestore-pv.libsonnet"; local instance = google_cloud_file_store_pv.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/prototypes/iap-ingress.jsonnet b/kubeflow/gcp/prototypes/iap-ingress.jsonnet similarity index 96% rename from kubeflow/core/prototypes/iap-ingress.jsonnet rename to kubeflow/gcp/prototypes/iap-ingress.jsonnet index 72ad6c477fe..16f956b9998 100644 --- a/kubeflow/core/prototypes/iap-ingress.jsonnet +++ b/kubeflow/gcp/prototypes/iap-ingress.jsonnet @@ -13,6 +13,6 @@ // @optionalParam oauthSecretName string kubeflow-oauth The name of the secret containing the OAuth client_id and client_secret. // @optionalParam privateGKECluster string false Is the k8s cluster a private GKE cluster -local iap = import "kubeflow/core/iap.libsonnet"; +local iap = import "kubeflow/gcp/iap.libsonnet"; local instance = iap.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/prototypes/metric-collector.jsonnet b/kubeflow/gcp/prototypes/metric-collector.jsonnet similarity index 91% rename from kubeflow/core/prototypes/metric-collector.jsonnet rename to kubeflow/gcp/prototypes/metric-collector.jsonnet index 597b7688856..52479e6cf25 100644 --- a/kubeflow/core/prototypes/metric-collector.jsonnet +++ b/kubeflow/gcp/prototypes/metric-collector.jsonnet @@ -8,6 +8,6 @@ // @optionalParam metricImage string gcr.io/kubeflow-images-public/metric-collector:latest Image for running metric exporter of kubeflow availability. // @optionalParam oauthSecretName string kubeflow-oauth The name of the secret containing the OAuth client_id and client_secret. -local metricCollector = import "kubeflow/core/metric-collector.libsonnet"; +local metricCollector = import "kubeflow/gcp/metric-collector.libsonnet"; local instance = metricCollector.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/prototypes/prometheus.jsonnet b/kubeflow/gcp/prototypes/prometheus.jsonnet similarity index 72% rename from kubeflow/core/prototypes/prometheus.jsonnet rename to kubeflow/gcp/prototypes/prometheus.jsonnet index abd41a5fce4..54fb37fbefb 100644 --- a/kubeflow/core/prototypes/prometheus.jsonnet +++ b/kubeflow/gcp/prototypes/prometheus.jsonnet @@ -1,12 +1,12 @@ // @apiVersion 0.1 // @name io.ksonnet.pkg.prometheus -// @description Provides prometheus prototype in kubeflow core. +// @description Provides prometheus prototype in kubeflow gcp package // @shortDescription Prometheus Service. // @param name string Name for the component // @param projectId string GCP project id. // @param clusterName string GKE cluster name. // @param zone string GKE cluster zone. -local prometheus = import "kubeflow/core/prometheus.libsonnet"; +local prometheus = import "kubeflow/gcp/prometheus.libsonnet"; local instance = prometheus.new(env, params); instance.list(instance.all) diff --git a/kubeflow/core/setup_backend.sh b/kubeflow/gcp/setup_backend.sh similarity index 100% rename from kubeflow/core/setup_backend.sh rename to kubeflow/gcp/setup_backend.sh diff --git a/kubeflow/core/tests/iap_test.jsonnet b/kubeflow/gcp/tests/iap_test.jsonnet similarity index 100% rename from kubeflow/core/tests/iap_test.jsonnet rename to kubeflow/gcp/tests/iap_test.jsonnet diff --git a/kubeflow/core/update_backend.sh b/kubeflow/gcp/update_backend.sh similarity index 100% rename from kubeflow/core/update_backend.sh rename to kubeflow/gcp/update_backend.sh diff --git a/scripts/gke/util.sh b/scripts/gke/util.sh index 92af643e50b..062f7752ba2 100644 --- a/scripts/gke/util.sh +++ b/scripts/gke/util.sh @@ -223,6 +223,10 @@ gcpGenerateKsApp() { pushd . cd "${KUBEFLOW_KS_DIR}" + # Install the gcp package + ks pkg install kubeflow/gcp + + # Generate all required components ks generate cloud-endpoints cloud-endpoints ks generate cert-manager cert-manager --acmeEmail=${EMAIL} ks generate iap-ingress iap-ingress --ipName=${KUBEFLOW_IP_NAME} --hostname=${KUBEFLOW_HOSTNAME} diff --git a/testing/deploy_utils.py b/testing/deploy_utils.py index 3ed14390190..ef3b4991dac 100644 --- a/testing/deploy_utils.py +++ b/testing/deploy_utils.py @@ -102,7 +102,7 @@ def setup_kubeflow_ks_app(dir, namespace, github_token, api_client): ["ks", "registry", "add", "kubeflow", kubeflow_registry], cwd=app_dir) # Install required packages - packages = ["kubeflow/core", "kubeflow/jupyter", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/tf-training", "kubeflow/pytorch-job", "kubeflow/argo"] + packages = ["kubeflow/core", "kubeflow/gcp", "kubeflow/jupyter", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/tf-training", "kubeflow/pytorch-job", "kubeflow/argo"] # Instead of installing packages we edit the app.yaml file directly #for p in packages: diff --git a/testing/test_jsonnet.py b/testing/test_jsonnet.py index 034738ff39e..136b7edb57c 100644 --- a/testing/test_jsonnet.py +++ b/testing/test_jsonnet.py @@ -22,7 +22,7 @@ Example invocation -python python -m testing.test_jsonnet --test_files_dirs=/kubeflow/application/tests,/kubeflow/core/tests,/kubeflow/jupyter/tests,/kubeflow/iap/tests,/kubeflow/tensorboard/tests,/kubeflow/examples/tests,/kubeflow/metacontroller/tests,/kubeflow/profiles/tests,/kubeflow/tf-training/tests --artifacts_dir=/tmp/artifacts +python python -m testing.test_jsonnet --test_files_dirs=/kubeflow/application/tests,/kubeflow/core/tests,/kubeflow/jupyter/tests,/kubeflow/iap/tests,/kubeflow/gcp/tests,/kubeflow/tensorboard/tests,/kubeflow/examples/tests,/kubeflow/metacontroller/tests,/kubeflow/profiles/tests,/kubeflow/tf-training/tests --artifacts_dir=/tmp/artifacts """