From 1004e635dd1a3b1cf9229e3916eefc6495ddb326 Mon Sep 17 00:00:00 2001 From: Yao Xiao Date: Tue, 20 Apr 2021 18:11:41 -0700 Subject: [PATCH] Add AWS 1.3 distribution (#1832) * Add AWS 1.3 distribution * Add kubeflow-apps * Add kfserving * Patch istio-ingressgateway as LoadBalancer * Fix AutoML Tensorboards Notebooks * Remove service * Change for katib --- distributions/kfdef/kfctl_aws.v1.3.0.yaml | 145 +++++++++++++ distributions/kfdef/kfctl_aws.yaml | 127 +++++++----- .../cert-manager-crds/kustomization.yaml | 5 - .../kustomization.yaml | 5 - .../cert-manager/kustomization.yaml | 14 -- .../kustomization.yaml | 5 - .../cluster-local-gateway/kustomization.yaml | 5 - .../centraldashboard_links_config.yaml | 67 ------ .../centraldashboard/kustomization.yaml | 10 - .../application/dex-auth/config/params.env | 2 - .../application/dex-auth/configmap_patch.yaml | 30 --- .../application/dex-auth/kustomization.yaml | 34 ---- .../istio-1-3-1-stack/kustomization.yaml | 6 - .../istio-1-9-1-stack/kustomization.yaml | 7 + .../istio-ingress-cognito/kustomization.yaml | 53 ----- .../istio-stack/kustomization.yaml | 6 - .../aws/application/istio/kustomization.yaml | 12 -- .../stacks/aws/application/istio/params.env | 1 - .../stacks/aws/application/istio/params.yaml | 3 - .../configs/spawner_ui_config.yaml | 190 ------------------ .../jupyter-web-app/kustomization.yaml | 10 - .../application/notebooks/kustomization.yaml | 6 + .../oidc-authservice/kustomization.yaml | 18 -- .../application/spartakus/kustomization.yaml | 10 - .../tensorboard/kustomization.yaml | 6 + distributions/stacks/aws/config/params.env | 5 - distributions/stacks/aws/kustomization.yaml | 64 ------ 27 files changed, 243 insertions(+), 603 deletions(-) create mode 100644 distributions/kfdef/kfctl_aws.v1.3.0.yaml delete mode 100644 distributions/stacks/aws/application/cert-manager-crds/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/cert-manager-kube-system-resources/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/cert-manager/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/cluster-local-gateway-1-3-1/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/cluster-local-gateway/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/common/centraldashboard/configs/centraldashboard_links_config.yaml delete mode 100644 distributions/stacks/aws/application/common/centraldashboard/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/dex-auth/config/params.env delete mode 100644 distributions/stacks/aws/application/dex-auth/configmap_patch.yaml delete mode 100644 distributions/stacks/aws/application/dex-auth/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/istio-1-3-1-stack/kustomization.yaml create mode 100644 distributions/stacks/aws/application/istio-1-9-1-stack/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/istio-ingress-cognito/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/istio-stack/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/istio/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/istio/params.env delete mode 100644 distributions/stacks/aws/application/istio/params.yaml delete mode 100644 distributions/stacks/aws/application/jupyter-web-app/configs/spawner_ui_config.yaml delete mode 100644 distributions/stacks/aws/application/jupyter-web-app/kustomization.yaml create mode 100644 distributions/stacks/aws/application/notebooks/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/oidc-authservice/kustomization.yaml delete mode 100644 distributions/stacks/aws/application/spartakus/kustomization.yaml create mode 100644 distributions/stacks/aws/application/tensorboard/kustomization.yaml delete mode 100644 distributions/stacks/aws/config/params.env delete mode 100644 distributions/stacks/aws/kustomization.yaml diff --git a/distributions/kfdef/kfctl_aws.v1.3.0.yaml b/distributions/kfdef/kfctl_aws.v1.3.0.yaml new file mode 100644 index 00000000000..4b1b1f5b08c --- /dev/null +++ b/distributions/kfdef/kfctl_aws.v1.3.0.yaml @@ -0,0 +1,145 @@ +apiVersion: kfdef.apps.kubeflow.org/v1 +kind: KfDef +metadata: + namespace: kubeflow +spec: + applications: + - kustomizeConfig: + repoRef: + name: manifests + path: common/kubeflow-namespace/base + name: kubeflow-namespace + - kustomizeConfig: + repoRef: + name: manifests + path: common/kubeflow-roles/base + name: kubeflow-roles + - kustomizeConfig: + repoRef: + name: manifests + path: distributions/stacks/aws/application/istio-1-9-1-stack + name: istio-stack + - kustomizeConfig: + repoRef: + name: manifests + path: common/cert-manager/cert-manager-kube-system-resources/base + name: cert-manager-kube-system-resources + - kustomizeConfig: + repoRef: + name: manifests + path: common/cert-manager/cert-manager-crds/base + name: cert-manager-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/cert-manager/cert-manager/overlays/self-signed + name: cert-manager + - kustomizeConfig: + repoRef: + name: manifests + path: common/dex/overlays/istio + name: dex + - kustomizeConfig: + repoRef: + name: manifests + path: common/oidc-authservice/base + name: oidc-authservice + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-serving-crds/base + name: knative-serving-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-serving-install/base + name: knative-serving-install + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-eventing-crds/base + name: knative-eventing-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-eventing-install/base + name: knative-eventing-install + - kustomizeConfig: + repoRef: + name: manifests + path: common/istio-1-9-0/kubeflow-istio-resources/base + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: apps/pipeline/upstream/env/platform-agnostic-multi-user + name: kubeflow-pipelines + - kustomizeConfig: + repoRef: + name: manifests + path: apps/kfserving/upstream/overlays/kubeflow + name: kfserving + - kustomizeConfig: + repoRef: + name: manifests + path: apps/katib/upstream/installs/katib-with-kubeflow + name: katib + - kustomizeConfig: + repoRef: + name: manifests + path: apps/centraldashboard/upstream/overlays/istio + name: central-dashboard + - kustomizeConfig: + repoRef: + name: manifests + path: distributions/stacks/aws/application/notebooks + name: notebooks + - kustomizeConfig: + repoRef: + name: manifests + path: apps/profiles/upstream/overlays/kubeflow + name: profiles + - kustomizeConfig: + repoRef: + name: manifests + path: apps/volumes-web-app/upstream/overlays/istio + name: volumes-web-app + - kustomizeConfig: + repoRef: + name: manifests + path: distributions/stacks/aws/application/tensorboard + name: tensorboard + - kustomizeConfig: + repoRef: + name: manifests + path: apps/tf-training/upstream/overlays/kubeflow + name: tfjob-operator + - kustomizeConfig: + repoRef: + name: manifests + path: apps/pytorch-job/upstream/overlays/kubeflow + name: pytorch-operator + - kustomizeConfig: + repoRef: + name: manifests + path: apps/mpi-job/upstream/overlays/kubeflow + name: mpi-operator + - kustomizeConfig: + repoRef: + name: manifests + path: apps/mxnet-job/upstream/overlays/kubeflow + name: mxnet-operator + - kustomizeConfig: + repoRef: + name: manifests + path: apps/xgboost-job/upstream/overlays/kubeflow + name: xgboost-operator + - kustomizeConfig: + repoRef: + name: manifests + path: common/user-namespace/base + name: user-namespace + repos: + - name: manifests + uri: https://github.com/kubeflow/manifests/archive/v1.3-branch.tar.gz + version: v1.3-branch diff --git a/distributions/kfdef/kfctl_aws.yaml b/distributions/kfdef/kfctl_aws.yaml index e2e71dfcd19..3b73b676dcb 100644 --- a/distributions/kfdef/kfctl_aws.yaml +++ b/distributions/kfdef/kfctl_aws.yaml @@ -7,107 +7,138 @@ spec: - kustomizeConfig: repoRef: name: manifests - path: namespaces/base - name: namespaces + path: common/kubeflow-namespace/base + name: kubeflow-namespace - kustomizeConfig: repoRef: name: manifests - path: application/v3 - name: application + path: common/kubeflow-roles/base + name: kubeflow-roles - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/istio-1-3-1-stack + path: distributions/stacks/aws/application/istio-1-9-1-stack name: istio-stack - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/cluster-local-gateway-1-3-1 + path: common/cert-manager/cert-manager-kube-system-resources/base + name: cert-manager-kube-system-resources + - kustomizeConfig: + repoRef: + name: manifests + path: common/cert-manager/cert-manager-crds/base + name: cert-manager-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/cert-manager/cert-manager/overlays/self-signed + name: cert-manager + - kustomizeConfig: + repoRef: + name: manifests + path: common/dex/overlays/istio + name: dex + - kustomizeConfig: + repoRef: + name: manifests + path: common/oidc-authservice/base + name: oidc-authservice + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-serving-crds/base + name: knative-serving-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-serving-install/base + name: knative-serving-install + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-eventing-crds/base + name: knative-eventing-crds + - kustomizeConfig: + repoRef: + name: manifests + path: common/knative/knative-eventing-install/base + name: knative-eventing-install + - kustomizeConfig: + repoRef: + name: manifests + path: common/istio-1-9-0/cluster-local-gateway/base name: cluster-local-gateway - kustomizeConfig: repoRef: name: manifests - path: istio/istio/base - name: istio + path: common/istio-1-9-0/kubeflow-istio-resources/base + name: kubeflow-istio-resources - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/cert-manager-crds - name: cert-manager-crds + path: apps/pipeline/upstream/env/platform-agnostic-multi-user + name: kubeflow-pipelines - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/cert-manager-kube-system-resources - name: cert-manager-kube-system-resources + path: apps/kfserving/upstream/overlays/kubeflow + name: kfserving - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/cert-manager - name: cert-manager + path: apps/centraldashboard/upstream/overlays/istio + name: central-dashboard - kustomizeConfig: repoRef: name: manifests - path: metacontroller/base - name: metacontroller + path: distributions/stacks/aws/application/notebooks + name: notebooks - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/oidc-authservice - name: oidc-authservice + path: apps/profiles/upstream/overlays/kubeflow + name: profiles - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/dex-auth - name: dex + path: apps/volumes-web-app/upstream/overlays/istio + name: volumes-web-app - kustomizeConfig: repoRef: name: manifests - path: admission-webhook/bootstrap/overlays/application - name: bootstrap + path: distributions/stacks/aws/application/tensorboard + name: tensorboard - kustomizeConfig: repoRef: name: manifests - path: spark/spark-operator/overlays/application - name: spark-operator + path: apps/tf-training/upstream/overlays/kubeflow + name: tfjob-operator - kustomizeConfig: repoRef: name: manifests - path: stacks/aws - name: kubeflow-apps + path: apps/pytorch-job/upstream/overlays/kubeflow + name: pytorch-operator - kustomizeConfig: repoRef: name: manifests - path: aws/istio-ingress/base_v3 - name: istio-ingress + path: apps/mpi-job/upstream/overlays/kubeflow + name: mpi-operator - kustomizeConfig: repoRef: name: manifests - path: knative/installs/generic - name: knative + path: apps/mxnet-job/upstream/overlays/kubeflow + name: mxnet-operator - kustomizeConfig: repoRef: name: manifests - path: kfserving/installs/generic - name: kfserving + path: apps/xgboost-job/upstream/overlays/kubeflow + name: xgboost-operator - kustomizeConfig: repoRef: name: manifests - path: stacks/aws/application/spartakus - name: spartakus - plugins: - - kind: KfAwsPlugin - metadata: - name: aws - spec: - auth: - basicAuth: - password: 12341234 - username: admin@kubeflow.org - region: us-west-2 - enablePodIamPolicy: true - # If you don't use IAM Role for Service Account, you can still use node instance roles. - #roles: - #- eksctl-kubeflow-aws-nodegroup-ng-a2-NodeInstanceRole-xxxxxxx + path: common/user-namespace/base + name: user-namespace repos: - name: manifests uri: https://github.com/kubeflow/manifests/archive/master.tar.gz diff --git a/distributions/stacks/aws/application/cert-manager-crds/kustomization.yaml b/distributions/stacks/aws/application/cert-manager-crds/kustomization.yaml deleted file mode 100644 index 7046d1031a3..00000000000 --- a/distributions/stacks/aws/application/cert-manager-crds/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: cert-manager -resources: -- ../../../../cert-manager/cert-manager-crds/base diff --git a/distributions/stacks/aws/application/cert-manager-kube-system-resources/kustomization.yaml b/distributions/stacks/aws/application/cert-manager-kube-system-resources/kustomization.yaml deleted file mode 100644 index 35d67006949..00000000000 --- a/distributions/stacks/aws/application/cert-manager-kube-system-resources/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kube-system -resources: -- ../../../../cert-manager/cert-manager-kube-system-resources/base diff --git a/distributions/stacks/aws/application/cert-manager/kustomization.yaml b/distributions/stacks/aws/application/cert-manager/kustomization.yaml deleted file mode 100644 index 58579b6269c..00000000000 --- a/distributions/stacks/aws/application/cert-manager/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -commonLabels: - app.kubernetes.io/component: cert-manager - app.kubernetes.io/name: cert-manager - kustomize.component: cert-manager -kind: Kustomization -namespace: cert-manager -resources: -- ../../../../cert-manager/cert-manager/base -- ../../../../cert-manager/cert-manager/overlays/application/application.yaml -- ../../../../cert-manager/cert-manager/overlays/self-signed/cluster-issuer.yaml -configurations: -- ../../../../cert-manager/cert-manager/overlays/application/params.yaml - diff --git a/distributions/stacks/aws/application/cluster-local-gateway-1-3-1/kustomization.yaml b/distributions/stacks/aws/application/cluster-local-gateway-1-3-1/kustomization.yaml deleted file mode 100644 index 7d5cd65106b..00000000000 --- a/distributions/stacks/aws/application/cluster-local-gateway-1-3-1/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../istio-1-3-1/cluster-local-gateway-1-3-1/base diff --git a/distributions/stacks/aws/application/cluster-local-gateway/kustomization.yaml b/distributions/stacks/aws/application/cluster-local-gateway/kustomization.yaml deleted file mode 100644 index 00913742046..00000000000 --- a/distributions/stacks/aws/application/cluster-local-gateway/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../istio/cluster-local-gateway/base diff --git a/distributions/stacks/aws/application/common/centraldashboard/configs/centraldashboard_links_config.yaml b/distributions/stacks/aws/application/common/centraldashboard/configs/centraldashboard_links_config.yaml deleted file mode 100644 index 5ddc7ad1fe5..00000000000 --- a/distributions/stacks/aws/application/common/centraldashboard/configs/centraldashboard_links_config.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: v1 -data: - links: |- - { - "menuLinks": [ - { - "link": "/pipeline/", - "text": "Pipelines" - }, - { - "link": "/jupyter/", - "text": "Notebook Servers" - }, - { - "link": "/katib/", - "text": "Katib" - } - ], - "externalLinks": [], - "quickLinks": [ - { - "text": "Upload a pipeline", - "desc": "Pipelines", - "link": "/pipeline/" - }, - { - "text": "View all pipeline runs", - "desc": "Pipelines", - "link": "/pipeline/#/runs" - }, - { - "text": "Create a new Notebook server", - "desc": "Notebook Servers", - "link": "/jupyter/new?namespace=kubeflow" - }, - { - "text": "View Katib Experiments", - "desc": "Katib", - "link": "/katib/" - } - ], - "documentationItems": [ - { - "text": "Getting Started with Kubeflow", - "desc": "Get your machine-learning workflow up and running on Kubeflow", - "link": "https://www.kubeflow.org/docs/started/getting-started/" - }, - { - "text": "EKS Kubeflow Examples", - "desc": "Get your step-by-step guidance for Kubeflow on AWS", - "link": "https://github.com/aws-samples/eks-kubeflow-workshop" - }, - { - "text": "Kubeflow on AWS", - "desc": "Running Kubeflow on Elastic Container Service and Amazon Web Services", - "link": "https://www.kubeflow.org/docs/aws/" - }, - { - "text": "Requirements for Kubeflow", - "desc": "Get more detailed information about using Kubeflow and its components", - "link": "https://www.kubeflow.org/docs/started/requirements/" - } - ] - } -kind: ConfigMap -metadata: - name: centraldashboard-links-config diff --git a/distributions/stacks/aws/application/common/centraldashboard/kustomization.yaml b/distributions/stacks/aws/application/common/centraldashboard/kustomization.yaml deleted file mode 100644 index 8ee0dd37bf7..00000000000 --- a/distributions/stacks/aws/application/common/centraldashboard/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- ../../../../../common/centraldashboard/overlays/stacks -configMapGenerator: -- name: centraldashboard-links-config - behavior: merge - files: - - ./configs/centraldashboard_links_config.yaml diff --git a/distributions/stacks/aws/application/dex-auth/config/params.env b/distributions/stacks/aws/application/dex-auth/config/params.env deleted file mode 100644 index 96b4d3fd964..00000000000 --- a/distributions/stacks/aws/application/dex-auth/config/params.env +++ /dev/null @@ -1,2 +0,0 @@ -static_email=admin@example.com -static_password_hash=$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W diff --git a/distributions/stacks/aws/application/dex-auth/configmap_patch.yaml b/distributions/stacks/aws/application/dex-auth/configmap_patch.yaml deleted file mode 100644 index 6d1ae171453..00000000000 --- a/distributions/stacks/aws/application/dex-auth/configmap_patch.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: dex -data: - config.yaml: | - issuer: $(issuer) - storage: - type: kubernetes - config: - inCluster: true - web: - http: 0.0.0.0:5556 - logger: - level: "debug" - format: text - oauth2: - skipApprovalScreen: true - enablePasswordDB: true - staticPasswords: - - email: $(config_static_email) - hash: $(config_static_password_hash) - username: $(static_username) - userID: $(static_user_id) - staticClients: - - id: $(client_id) - redirectURIs: $(oidc_redirect_uris) - name: 'Dex Login Application' - secret: $(application_secret) diff --git a/distributions/stacks/aws/application/dex-auth/kustomization.yaml b/distributions/stacks/aws/application/dex-auth/kustomization.yaml deleted file mode 100644 index bff1ef45a08..00000000000 --- a/distributions/stacks/aws/application/dex-auth/kustomization.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: auth -resources: -- ../../../../dex-auth/dex-crds/overlays/istio -configMapGenerator: -- name: dex-parameters - behavior: merge - literals: - - issuer="http://dex.auth.svc.cluster.local:5556/dex" - - oidc_redirect_uris=["/login/oidc"] - - client_id="kubeflow-oidc-authservice" -- envs: - - ./config/params.env - name: dex-config -vars: -- name: config_static_email - objref: - kind: ConfigMap - name: dex-config - apiVersion: v1 - fieldref: - fieldpath: data.static_email -- name: config_static_password_hash - objref: - kind: ConfigMap - name: dex-config - apiVersion: v1 - fieldref: - fieldpath: data.static_password_hash -patchesStrategicMerge: -- configmap_patch.yaml -generatorOptions: - disableNameSuffixHash: true diff --git a/distributions/stacks/aws/application/istio-1-3-1-stack/kustomization.yaml b/distributions/stacks/aws/application/istio-1-3-1-stack/kustomization.yaml deleted file mode 100644 index 47428640e29..00000000000 --- a/distributions/stacks/aws/application/istio-1-3-1-stack/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../istio-1-3-1/istio-crds-1-3-1/base -- ../../../../istio-1-3-1/istio-install-1-3-1/base diff --git a/distributions/stacks/aws/application/istio-1-9-1-stack/kustomization.yaml b/distributions/stacks/aws/application/istio-1-9-1-stack/kustomization.yaml new file mode 100644 index 00000000000..4112ee099fb --- /dev/null +++ b/distributions/stacks/aws/application/istio-1-9-1-stack/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: istio-system +resources: +- ../../../../../common/istio-1-9-0/istio-crds/base +- ../../../../../common/istio-1-9-0/istio-namespace/base +- ../../../../../common/istio-1-9-0/istio-install/base \ No newline at end of file diff --git a/distributions/stacks/aws/application/istio-ingress-cognito/kustomization.yaml b/distributions/stacks/aws/application/istio-ingress-cognito/kustomization.yaml deleted file mode 100644 index c1f57ff8bc7..00000000000 --- a/distributions/stacks/aws/application/istio-ingress-cognito/kustomization.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../aws/istio-ingress/base/ingress.yaml -commonLabels: - kustomize.component: istio-ingress -patchesStrategicMerge: -- ../../../../aws/istio-ingress/overlays/cognito/ingress.yaml -generatorOptions: - disableNameSuffixHash: true -configMapGenerator: -- envs: - - ../../../../aws/istio-ingress/overlays/cognito/params.env - name: istio-ingress-config -vars: -- name: loadBalancerScheme - objref: - kind: ConfigMap - name: istio-ingress-config - apiVersion: v1 - fieldref: - fieldpath: data.loadBalancerScheme -- name: CognitoUserPoolArn - objref: - kind: ConfigMap - name: istio-ingress-config - apiVersion: v1 - fieldref: - fieldpath: data.CognitoUserPoolArn -- name: CognitoAppClientId - objref: - kind: ConfigMap - name: istio-ingress-config - apiVersion: v1 - fieldref: - fieldpath: data.CognitoAppClientId -- name: CognitoUserPoolDomain - objref: - kind: ConfigMap - name: istio-ingress-config - apiVersion: v1 - fieldref: - fieldpath: data.CognitoUserPoolDomain -- name: certArn - objref: - kind: ConfigMap - name: istio-ingress-config - apiVersion: v1 - fieldref: - fieldpath: data.certArn -configurations: -- ../../../../aws/istio-ingress/overlays/cognito/params.yaml diff --git a/distributions/stacks/aws/application/istio-stack/kustomization.yaml b/distributions/stacks/aws/application/istio-stack/kustomization.yaml deleted file mode 100644 index ca325cebbe3..00000000000 --- a/distributions/stacks/aws/application/istio-stack/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../istio/istio-crds/base -- ../../../../istio/istio-install/base diff --git a/distributions/stacks/aws/application/istio/kustomization.yaml b/distributions/stacks/aws/application/istio/kustomization.yaml deleted file mode 100644 index 3552e93675f..00000000000 --- a/distributions/stacks/aws/application/istio/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- ../../../../istio/istio/base -configMapGenerator: -- name: istio-parameters - behavior: merge - envs: - - params.env -configurations: -- params.yaml diff --git a/distributions/stacks/aws/application/istio/params.env b/distributions/stacks/aws/application/istio/params.env deleted file mode 100644 index 99591d647ca..00000000000 --- a/distributions/stacks/aws/application/istio/params.env +++ /dev/null @@ -1 +0,0 @@ -clusterRbacConfig=ON diff --git a/distributions/stacks/aws/application/istio/params.yaml b/distributions/stacks/aws/application/istio/params.yaml deleted file mode 100644 index 87a69bd6609..00000000000 --- a/distributions/stacks/aws/application/istio/params.yaml +++ /dev/null @@ -1,3 +0,0 @@ -varReference: - - path: spec/mode - kind: ClusterRbacConfig diff --git a/distributions/stacks/aws/application/jupyter-web-app/configs/spawner_ui_config.yaml b/distributions/stacks/aws/application/jupyter-web-app/configs/spawner_ui_config.yaml deleted file mode 100644 index 5924d5c3624..00000000000 --- a/distributions/stacks/aws/application/jupyter-web-app/configs/spawner_ui_config.yaml +++ /dev/null @@ -1,190 +0,0 @@ -# Configuration file for the Jupyter UI. -# -# Each Jupyter UI option is configured by two keys: 'value' and 'readOnly' -# - The 'value' key contains the default value -# - The 'readOnly' key determines if the option will be available to users -# -# If the 'readOnly' key is present and set to 'true', the respective option -# will be disabled for users and only set by the admin. Also when a -# Notebook is POSTED to the API if a necessary field is not present then -# the value from the config will be used. -# -# If the 'readOnly' key is missing (defaults to 'false'), the respective option -# will be available for users to edit. -# -# Note that some values can be templated. Such values are the names of the -# Volumes as well as their StorageClass -spawnerFormDefaults: - image: - # The container Image for the user's Jupyter Notebook - # If readonly, this value must be a member of the list below - value: 527798164940.dkr.ecr.us-west-2.amazonaws.com/tensorflow-1.15.2-notebook-cpu:1.2.0 - # The list of available standard container Images - options: - - 527798164940.dkr.ecr.us-west-2.amazonaws.com/tensorflow-1.15.2-notebook-cpu:1.2.0 - - 527798164940.dkr.ecr.us-west-2.amazonaws.com/tensorflow-1.15.2-notebook-gpu:1.2.0 - - 527798164940.dkr.ecr.us-west-2.amazonaws.com/tensorflow-2.1.0-notebook-cpu:1.2.0 - - 527798164940.dkr.ecr.us-west-2.amazonaws.com/tensorflow-2.1.0-notebook-gpu:1.2.0 - - gcr.io/kubeflow-images-public/tensorflow-1.15.2-notebook-cpu:1.0.0 - - gcr.io/kubeflow-images-public/tensorflow-1.15.2-notebook-gpu:1.0.0 - - gcr.io/kubeflow-images-public/tensorflow-2.1.0-notebook-cpu:1.0.0 - - gcr.io/kubeflow-images-public/tensorflow-2.1.0-notebook-gpu:1.0.0 - # By default, custom container Images are allowed - # Uncomment the following line to only enable standard container Images - readOnly: false - cpu: - # CPU for user's Notebook - value: '0.5' - readOnly: false - memory: - # Memory for user's Notebook - value: 1.0Gi - readOnly: false - workspaceVolume: - # Workspace Volume to be attached to user's Notebook - # Each Workspace Volume is declared with the following attributes: - # Type, Name, Size, MountPath and Access Mode - value: - type: - # The Type of the Workspace Volume - # Supported values: 'New', 'Existing' - value: New - name: - # The Name of the Workspace Volume - # Note that this is a templated value. Special values: - # {notebook-name}: Replaced with the name of the Notebook. The frontend - # will replace this value as the user types the name - value: 'workspace-{notebook-name}' - size: - # The Size of the Workspace Volume (in Gi) - value: '10Gi' - mountPath: - # The Path that the Workspace Volume will be mounted - value: /home/jovyan - accessModes: - # The Access Mode of the Workspace Volume - # Supported values: 'ReadWriteOnce', 'ReadWriteMany', 'ReadOnlyMany' - value: ReadWriteOnce - class: - # The StrageClass the PVC will use if type is New. Special values are: - # {none}: default StorageClass - # {empty}: empty string "" - value: '{none}' - readOnly: false - dataVolumes: - # List of additional Data Volumes to be attached to the user's Notebook - value: [] - # Each Data Volume is declared with the following attributes: - # Type, Name, Size, MountPath and Access Mode - # - # For example, a list with 2 Data Volumes: - # value: - # - value: - # type: - # value: New - # name: - # value: '{notebook-name}-vol-1' - # size: - # value: '10Gi' - # class: - # value: standard - # mountPath: - # value: /home/jovyan/vol-1 - # accessModes: - # value: ReadWriteOnce - # class: - # value: {none} - # - value: - # type: - # value: New - # name: - # value: '{notebook-name}-vol-2' - # size: - # value: '10Gi' - # mountPath: - # value: /home/jovyan/vol-2 - # accessModes: - # value: ReadWriteMany - # class: - # value: {none} - readOnly: false - gpus: - # Number of GPUs to be assigned to the Notebook Container - value: - # values: "none", "1", "2", "4", "8" - num: "none" - # Determines what the UI will show and send to the backend - vendors: - - limitsKey: "nvidia.com/gpu" - uiName: "NVIDIA" - - limitsKey: "amd.com/gpu" - uiName: "AMD" - # Values: "" or a `limits-key` from the vendors list - vendor: "" - readOnly: false - shm: - value: true - readOnly: false - configurations: - # List of labels to be selected, these are the labels from PodDefaults - # value: - # - add-gcp-secret - # - default-editor - value: [] - readOnly: false - affinityConfig: - # The default `configKey` from the options list - # If readonly, the default value will be the only option - value: "none" - # The list of available affinity configs - options: - - configKey: "none" - displayName: "None" - affinity: {} - # # (DESC) Pod gets an exclusive "n1-standard-2" Node - # # (TIP) set PreferNoSchedule taint on this node-pool - # # (TIP) enable cluster-autoscaler on this node-pool - # # (TIP) dont let users request more CPU/MEMORY than the size of this node - # - configKey: "exclusive__n1-standard-2" - # displayName: "Exclusive: n1-standard-2" - # affinity: - # # (Require) Node having label: `node_pool=notebook-n1-standard-2` - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: "node_pool" - # operator: "In" - # values: - # - "notebook-n1-standard-2" - # # (Require) Node WITHOUT existing Pod having label: `notebook-name` - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - labelSelector: - # matchExpressions: - # - key: "notebook-name" - # operator: "Exists" - # namespaces: [] - # topologyKey: "kubernetes.io/hostname" - readOnly: false - tolerationGroup: - # The default `groupKey` from the options list - # If readonly, the default value will be the only option - value: "none" - # The list of available tolerationGroup configs - options: - - groupKey: "none" - displayName: "None" - tolerations: [] - # - groupKey: "group_1" - # displayName: "Group 1: description" - # tolerations: - # - key: "key1" - # operator: "Equal" - # value: "value1" - # effect: "NoSchedule" - # - key: "key2" - # operator: "Equal" - # value: "value2" - # effect: "NoSchedule" - readOnly: false \ No newline at end of file diff --git a/distributions/stacks/aws/application/jupyter-web-app/kustomization.yaml b/distributions/stacks/aws/application/jupyter-web-app/kustomization.yaml deleted file mode 100644 index 9b270080f4d..00000000000 --- a/distributions/stacks/aws/application/jupyter-web-app/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- ../../../../jupyter/jupyter-web-app/base_v3 -configMapGenerator: -- name: jupyter-web-app-config - behavior: merge - files: - - ./configs/spawner_ui_config.yaml diff --git a/distributions/stacks/aws/application/notebooks/kustomization.yaml b/distributions/stacks/aws/application/notebooks/kustomization.yaml new file mode 100644 index 00000000000..22cd94da0a6 --- /dev/null +++ b/distributions/stacks/aws/application/notebooks/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +resources: +- ../../../../../apps/jupyter/notebook-controller/upstream/overlays/kubeflow +- ../../../../../apps/jupyter/jupyter-web-app/upstream/overlays/istio diff --git a/distributions/stacks/aws/application/oidc-authservice/kustomization.yaml b/distributions/stacks/aws/application/oidc-authservice/kustomization.yaml deleted file mode 100644 index e8a8eade1f3..00000000000 --- a/distributions/stacks/aws/application/oidc-authservice/kustomization.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: istio-system -resources: -- ../../../../istio/oidc-authservice/overlays/application -configMapGenerator: -- name: oidc-authservice-parameters - behavior: merge - literals: - - client_id=kubeflow-oidc-authservice - - oidc_provider=http://dex.auth.svc.cluster.local:5556/dex - - oidc_redirect_uri=/login/oidc - - oidc_auth_url=/dex/auth - - application_secret=pUBnBOY80SnXgjibTYM9ZWNzY2xreNGQok - - skip_auth_uri=/dex - - namespace=istio-system - - userid-header=kubeflow-userid - - userid-prefix= diff --git a/distributions/stacks/aws/application/spartakus/kustomization.yaml b/distributions/stacks/aws/application/spartakus/kustomization.yaml deleted file mode 100644 index 10560182cdd..00000000000 --- a/distributions/stacks/aws/application/spartakus/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- ../../../../common/spartakus/overlays/application -configMapGenerator: -- name: spartakus-config - behavior: merge - literals: - - usageId= diff --git a/distributions/stacks/aws/application/tensorboard/kustomization.yaml b/distributions/stacks/aws/application/tensorboard/kustomization.yaml new file mode 100644 index 00000000000..bee49f91cdd --- /dev/null +++ b/distributions/stacks/aws/application/tensorboard/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +resources: +- ../../../../../apps/tensorboard/tensorboards-web-app/upstream/overlays/istio +- ../../../../../apps/tensorboard/tensorboard-controller/upstream/overlays/kubeflow \ No newline at end of file diff --git a/distributions/stacks/aws/config/params.env b/distributions/stacks/aws/config/params.env deleted file mode 100644 index e296f8f01bd..00000000000 --- a/distributions/stacks/aws/config/params.env +++ /dev/null @@ -1,5 +0,0 @@ -clusterDomain=cluster.local -userid-header=kubeflow-userid -userid-prefix= -cluster-name= -istio-namespace=istio-system diff --git a/distributions/stacks/aws/kustomization.yaml b/distributions/stacks/aws/kustomization.yaml deleted file mode 100644 index 596979754c5..00000000000 --- a/distributions/stacks/aws/kustomization.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: - # List the Kubeflow applications that should be included - # control plane - - ../../admission-webhook/webhook/v3 - - ./application/common/centraldashboard - - ../../kubeflow-roles/base - - ./application/jupyter-web-app - - ../../jupyter/notebook-controller/base_v3 - - ../../profiles/base_v3 - # Training Operators - - ../../pytorch-job/pytorch-job-crds/overlays/application - - ../../pytorch-job/pytorch-operator/overlays/application - - ../../tf-training/tf-job-crds/overlays/application - - ../../tf-training/tf-job-operator/overlays/application - - ../../mxnet-job/mxnet-operator/overlays/application - - ../../mpi-job/mpi-operator/overlays/application - - ../../katib/installs/katib-standalone - # Pipeline - - ../../pipeline/minio/installs/generic - - ../../pipeline/mysql/installs/generic - - ../../pipeline/installs/multi-user - - ../../argo/base_v3 - # Serving components - - ../../seldon/seldon-core-operator/overlays/application - # AWS components - - ../../aws/aws-alb-ingress-controller/base_v3 - - ../../aws/nvidia-device-plugin/overlays/application - # Kubernetes resources - anonymous user - - ../../default-install/base - # Metadata - - ../../metadata/v3 -generatorOptions: - disableNameSuffixHash: true -configMapGenerator: -- envs: - - ./config/params.env - name: kubeflow-config -vars: -# We need to define vars at the top level otherwise we will get -# conflicts. -- fieldref: - fieldPath: data.clusterDomain - name: clusterDomain - objref: - apiVersion: v1 - kind: ConfigMap - name: kubeflow-config -- fieldref: - fieldPath: metadata.namespace - name: namespace - objref: - apiVersion: v1 - kind: ConfigMap - name: kubeflow-config -- fieldref: - fieldpath: metadata.namespace - name: katib-ui-namespace - objref: - kind: Service - name: katib-ui - apiVersion: v1