From 3d8651eda5aed2ca15522cba2c986a4ab0f94a38 Mon Sep 17 00:00:00 2001 From: "Yuan (Bob) Gong" Date: Tue, 7 Jul 2020 22:57:58 +0800 Subject: [PATCH] feat: KFP multi user mode PR1 - enable multi user mode without istio authorization (#1342) * Add argo to stacks/generic * Pull pipelines manifest from upstream * Updated kfp * Minio v3 manifests * Rename minio configmap * Add generic minio install * Generate new test data * Mysql kustomize v3 manifest - generic install * Add mysql gcp pd install * Generate test data * Pipelines kustomize v3 manifests * Add kfp ui virtual service * Add metadata deployment to stacks/generic * Use common cluster domain * Deploy metadata writer * Add kfp cache server * Update test data * Enable KFP multi user mode without istio security * Fix persistence agent watch namespace * Fix namespace env for some deployments * Fix cluster roles and bindings * fix rename * Fix pipelines ui role * Updated kfp to rc2 * simplify pipeline v3 manifest using updated kfp rc2 manifest * Fix pipeline-install-config * remove redundant configmap * update tests * updated to kfp 1.0.0-rc.3 * Adapt to kfp 1.0rc3 refactoring * update test snapshots * fix pull kfp script to detect empty dir * fix example ref * update snapshot * fix gcp pd manifest * Update stacks ref * revert alice example to gcp stack * update snapshot * fix profile controller iam binding * Update kfp profile controller can be configured to different images and istio sidecar * add missing viewer controller cluster roles * Use python3 for sync.py * Revert gcp stack back to use non multi user kfp * revert unintended changes * revert upstream changes --- hack/pull_kfp_upstream.sh | 12 +- .../api-service/cluster-role-binding.yaml | 11 + .../multi-user/api-service/cluster-role.yaml | 34 +++ .../api-service/deployment-patch.yaml | 12 + .../multi-user/api-service/kustomization.yaml | 8 + .../multi-user/api-service/params.env | 4 + .../cache/cluster-role-binding.yaml | 11 + .../multi-user/cache/cluster-role.yaml | 31 ++ .../multi-user/cache/deployment-patch.yaml | 13 + .../multi-user/cache/kustomization.yaml | 5 + .../installs/multi-user/kustomization.yaml | 21 ++ .../metadata-writer/cluster-role-binding.yaml | 11 + .../metadata-writer/cluster-role.yaml | 31 ++ .../metadata-writer/deployment-patch.yaml | 13 + .../metadata-writer/kustomization.yaml | 5 + .../cluster-role-binding.yaml | 11 + .../persistence-agent/cluster-role.yaml | 21 ++ .../persistence-agent/deployment-patch.yaml | 13 + .../persistence-agent/kustomization.yaml | 5 + .../composite-controller.yaml | 43 +++ .../deployment.yaml | 27 ++ .../kustomization.yaml | 17 ++ .../pipelines-profile-controller/params.env | 3 + .../pipelines-profile-controller/service.yaml | 10 + .../pipelines-profile-controller/sync.py | 274 ++++++++++++++++++ .../pipelines-ui/cluster-role-binding.yaml | 11 + .../multi-user/pipelines-ui/cluster-role.yaml | 42 +++ .../multi-user/pipelines-ui/configmap.yaml | 13 + .../pipelines-ui/deployment-patch.yaml | 30 ++ .../pipelines-ui/kustomization.yaml | 9 + .../cluster-role-binding.yaml | 11 + .../scheduled-workflow/cluster-role.yaml | 36 +++ .../scheduled-workflow/deployment-patch.yaml | 13 + .../scheduled-workflow/kustomization.yaml | 6 + .../cluster-role-binding.yaml | 11 + .../viewer-controller/cluster-role.yaml | 30 ++ .../viewer-controller/deployment-patch.yaml | 13 + .../viewer-controller/kustomization.yaml | 5 + 38 files changed, 872 insertions(+), 4 deletions(-) create mode 100644 pipeline/installs/multi-user/api-service/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/api-service/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/api-service/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/api-service/kustomization.yaml create mode 100644 pipeline/installs/multi-user/api-service/params.env create mode 100644 pipeline/installs/multi-user/cache/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/cache/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/cache/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/cache/kustomization.yaml create mode 100644 pipeline/installs/multi-user/kustomization.yaml create mode 100644 pipeline/installs/multi-user/metadata-writer/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/metadata-writer/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/metadata-writer/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/metadata-writer/kustomization.yaml create mode 100644 pipeline/installs/multi-user/persistence-agent/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/persistence-agent/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/persistence-agent/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/persistence-agent/kustomization.yaml create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/composite-controller.yaml create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/deployment.yaml create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/kustomization.yaml create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/params.env create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/service.yaml create mode 100644 pipeline/installs/multi-user/pipelines-profile-controller/sync.py create mode 100644 pipeline/installs/multi-user/pipelines-ui/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/pipelines-ui/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/pipelines-ui/configmap.yaml create mode 100644 pipeline/installs/multi-user/pipelines-ui/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/pipelines-ui/kustomization.yaml create mode 100644 pipeline/installs/multi-user/scheduled-workflow/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/scheduled-workflow/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/scheduled-workflow/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/scheduled-workflow/kustomization.yaml create mode 100644 pipeline/installs/multi-user/viewer-controller/cluster-role-binding.yaml create mode 100644 pipeline/installs/multi-user/viewer-controller/cluster-role.yaml create mode 100644 pipeline/installs/multi-user/viewer-controller/deployment-patch.yaml create mode 100644 pipeline/installs/multi-user/viewer-controller/kustomization.yaml diff --git a/hack/pull_kfp_upstream.sh b/hack/pull_kfp_upstream.sh index d584325ee69..0d986871df2 100755 --- a/hack/pull_kfp_upstream.sh +++ b/hack/pull_kfp_upstream.sh @@ -10,11 +10,15 @@ set -ex # pipelines version. export PIPELINES_VERSION=1.0.0-rc.3 export PIPELINES_SRC_REPO=https://github.com/kubeflow/pipelines.git -# Pulling for the first time -# kpt pkg get $PIPELINES_SRC_REPO/manifests/kustomize@$PIPELINES_VERSION pipeline/upstream -# Updates -kpt pkg update pipeline/upstream/@$PIPELINES_VERSION --strategy force-delete-replace +if [ -d pipeline/upstream ]; then + # Updates + kpt pkg update pipeline/upstream/@$PIPELINES_VERSION --strategy force-delete-replace +else + # Pulling for the first time + kpt pkg get $PIPELINES_SRC_REPO/manifests/kustomize@$PIPELINES_VERSION pipeline/upstream +fi + # Before kubeflow/pipelines/manifests/kustomize supports kustomize v3.5+, we # have to convert kustomization.yaml env to envs syntax, so that it is compatible # with latest kustomize used in kubeflow/manifests. diff --git a/pipeline/installs/multi-user/api-service/cluster-role-binding.yaml b/pipeline/installs/multi-user/api-service/cluster-role-binding.yaml new file mode 100644 index 00000000000..9927d3e1005 --- /dev/null +++ b/pipeline/installs/multi-user/api-service/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline +subjects: +- kind: ServiceAccount + name: ml-pipeline diff --git a/pipeline/installs/multi-user/api-service/cluster-role.yaml b/pipeline/installs/multi-user/api-service/cluster-role.yaml new file mode 100644 index 00000000000..a88f27ff9e1 --- /dev/null +++ b/pipeline/installs/multi-user/api-service/cluster-role.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: ml-pipeline +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - update + - patch + - delete +- apiGroups: + - "" + resources: + - pods + verbs: + - delete diff --git a/pipeline/installs/multi-user/api-service/deployment-patch.yaml b/pipeline/installs/multi-user/api-service/deployment-patch.yaml new file mode 100644 index 00000000000..25a4e686a64 --- /dev/null +++ b/pipeline/installs/multi-user/api-service/deployment-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline +spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + envFrom: + - configMapRef: + name: pipeline-api-server-config diff --git a/pipeline/installs/multi-user/api-service/kustomization.yaml b/pipeline/installs/multi-user/api-service/kustomization.yaml new file mode 100644 index 00000000000..ce24f2ab364 --- /dev/null +++ b/pipeline/installs/multi-user/api-service/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role-binding.yaml +- cluster-role.yaml +configMapGenerator: +- name: pipeline-api-server-config + env: params.env diff --git a/pipeline/installs/multi-user/api-service/params.env b/pipeline/installs/multi-user/api-service/params.env new file mode 100644 index 00000000000..5bb1e0a3e99 --- /dev/null +++ b/pipeline/installs/multi-user/api-service/params.env @@ -0,0 +1,4 @@ +MULTIUSER=true +DEFAULTPIPELINERUNNERSERVICEACCOUNT=default-editor +VISUALIZATIONSERVICE_NAME=ml-pipeline-visualizationserver +VISUALIZATIONSERVICE_PORT=8888 diff --git a/pipeline/installs/multi-user/cache/cluster-role-binding.yaml b/pipeline/installs/multi-user/cache/cluster-role-binding.yaml new file mode 100644 index 00000000000..4e80257c20d --- /dev/null +++ b/pipeline/installs/multi-user/cache/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflow-pipelines-cache-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-pipelines-cache-role +subjects: +- kind: ServiceAccount + name: kubeflow-pipelines-cache diff --git a/pipeline/installs/multi-user/cache/cluster-role.yaml b/pipeline/installs/multi-user/cache/cluster-role.yaml new file mode 100644 index 00000000000..e604367357f --- /dev/null +++ b/pipeline/installs/multi-user/cache/cluster-role.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-pipelines-cache-role +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch diff --git a/pipeline/installs/multi-user/cache/deployment-patch.yaml b/pipeline/installs/multi-user/cache/deployment-patch.yaml new file mode 100644 index 00000000000..5f98ee136f7 --- /dev/null +++ b/pipeline/installs/multi-user/cache/deployment-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cache-server +spec: + template: + spec: + containers: + - name: server + env: + - name: NAMESPACE_TO_WATCH + value: '' + valueFrom: null diff --git a/pipeline/installs/multi-user/cache/kustomization.yaml b/pipeline/installs/multi-user/cache/kustomization.yaml new file mode 100644 index 00000000000..b1f65469e1d --- /dev/null +++ b/pipeline/installs/multi-user/cache/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/pipeline/installs/multi-user/kustomization.yaml b/pipeline/installs/multi-user/kustomization.yaml new file mode 100644 index 00000000000..8d3aef475f0 --- /dev/null +++ b/pipeline/installs/multi-user/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +resources: +- ../generic +- api-service +- pipelines-ui +- pipelines-profile-controller +- scheduled-workflow +- viewer-controller +- persistence-agent +- cache +- metadata-writer +patchesStrategicMerge: +- api-service/deployment-patch.yaml +- pipelines-ui/deployment-patch.yaml +- scheduled-workflow/deployment-patch.yaml +- viewer-controller/deployment-patch.yaml +- persistence-agent/deployment-patch.yaml +- metadata-writer/deployment-patch.yaml +- cache/deployment-patch.yaml diff --git a/pipeline/installs/multi-user/metadata-writer/cluster-role-binding.yaml b/pipeline/installs/multi-user/metadata-writer/cluster-role-binding.yaml new file mode 100644 index 00000000000..605f1ff0df7 --- /dev/null +++ b/pipeline/installs/multi-user/metadata-writer/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflow-pipelines-metadata-writer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-pipelines-metadata-writer-role +subjects: +- kind: ServiceAccount + name: kubeflow-pipelines-metadata-writer diff --git a/pipeline/installs/multi-user/metadata-writer/cluster-role.yaml b/pipeline/installs/multi-user/metadata-writer/cluster-role.yaml new file mode 100644 index 00000000000..a6ec9867253 --- /dev/null +++ b/pipeline/installs/multi-user/metadata-writer/cluster-role.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-pipelines-metadata-writer-role +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - update + - patch diff --git a/pipeline/installs/multi-user/metadata-writer/deployment-patch.yaml b/pipeline/installs/multi-user/metadata-writer/deployment-patch.yaml new file mode 100644 index 00000000000..2babe9f43fe --- /dev/null +++ b/pipeline/installs/multi-user/metadata-writer/deployment-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-writer +spec: + template: + spec: + containers: + - name: main + env: + - name: NAMESPACE_TO_WATCH + value: '' + valueFrom: null diff --git a/pipeline/installs/multi-user/metadata-writer/kustomization.yaml b/pipeline/installs/multi-user/metadata-writer/kustomization.yaml new file mode 100644 index 00000000000..b1f65469e1d --- /dev/null +++ b/pipeline/installs/multi-user/metadata-writer/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/pipeline/installs/multi-user/persistence-agent/cluster-role-binding.yaml b/pipeline/installs/multi-user/persistence-agent/cluster-role-binding.yaml new file mode 100644 index 00000000000..e030bd8a015 --- /dev/null +++ b/pipeline/installs/multi-user/persistence-agent/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-persistenceagent-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-persistenceagent-role +subjects: +- kind: ServiceAccount + name: ml-pipeline-persistenceagent diff --git a/pipeline/installs/multi-user/persistence-agent/cluster-role.yaml b/pipeline/installs/multi-user/persistence-agent/cluster-role.yaml new file mode 100644 index 00000000000..b3053317b53 --- /dev/null +++ b/pipeline/installs/multi-user/persistence-agent/cluster-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-persistenceagent-role +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - get + - list + - watch diff --git a/pipeline/installs/multi-user/persistence-agent/deployment-patch.yaml b/pipeline/installs/multi-user/persistence-agent/deployment-patch.yaml new file mode 100644 index 00000000000..1e165def422 --- /dev/null +++ b/pipeline/installs/multi-user/persistence-agent/deployment-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline-persistenceagent +spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: NAMESPACE + value: '' + valueFrom: null diff --git a/pipeline/installs/multi-user/persistence-agent/kustomization.yaml b/pipeline/installs/multi-user/persistence-agent/kustomization.yaml new file mode 100644 index 00000000000..b1f65469e1d --- /dev/null +++ b/pipeline/installs/multi-user/persistence-agent/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/composite-controller.yaml b/pipeline/installs/multi-user/pipelines-profile-controller/composite-controller.yaml new file mode 100644 index 00000000000..901d0787ac3 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/composite-controller.yaml @@ -0,0 +1,43 @@ +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: profile-controller +spec: + generateSelector: true + resyncPeriodSeconds: 10 + parentResource: + apiVersion: v1 + resource: namespaces + childResources: + - apiVersion: v1 + resource: secrets + updateStrategy: + method: OnDelete + - apiVersion: v1 + resource: configmaps + updateStrategy: + method: OnDelete + - apiVersion: apps/v1 + resource: deployments + updateStrategy: + method: InPlace + - apiVersion: v1 + resource: services + updateStrategy: + method: InPlace + - apiVersion: networking.istio.io/v1alpha3 + resource: destinationrules + updateStrategy: + method: InPlace + - apiVersion: rbac.istio.io/v1alpha1 + resource: serviceroles + updateStrategy: + method: InPlace + - apiVersion: rbac.istio.io/v1alpha1 + resource: servicerolebindings + updateStrategy: + method: InPlace + hooks: + sync: + webhook: + url: http://kubeflow-pipelines-profile-controller/sync diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/deployment.yaml b/pipeline/installs/multi-user/pipelines-profile-controller/deployment.yaml new file mode 100644 index 00000000000..0ccf4502892 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/deployment.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: profile-controller +spec: + replicas: 1 + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: profile-controller + image: python:3.7 + command: ["python", "/hooks/sync.py"] + envFrom: + - configMapRef: + name: profile-controller-env + volumeMounts: + - name: hooks + mountPath: /hooks + ports: + - containerPort: 80 + volumes: + - name: hooks + configMap: + name: profile-controller-code diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/kustomization.yaml b/pipeline/installs/multi-user/pipelines-profile-controller/kustomization.yaml new file mode 100644 index 00000000000..76a257965e1 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +nameprefix: kubeflow-pipelines- +commonLabels: + app: kubeflow-pipelines-profile-controller +resources: +- service.yaml +- deployment.yaml +- composite-controller.yaml +configMapGenerator: +- name: profile-controller-code + files: + - sync.py +- name: profile-controller-env + envs: + - params.env diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/params.env b/pipeline/installs/multi-user/pipelines-profile-controller/params.env new file mode 100644 index 00000000000..49e9aedae8d --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/params.env @@ -0,0 +1,3 @@ +KFP_VERSION=1.0.0-rc.3 +# TODO: make visualization server work with sidecar +DISABLE_ISTIO_SIDECAR=true diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/service.yaml b/pipeline/installs/multi-user/pipelines-profile-controller/service.yaml new file mode 100644 index 00000000000..d4df4dc2161 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: profile-controller +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 80 diff --git a/pipeline/installs/multi-user/pipelines-profile-controller/sync.py b/pipeline/installs/multi-user/pipelines-profile-controller/sync.py new file mode 100644 index 00000000000..8604ef4ef61 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-profile-controller/sync.py @@ -0,0 +1,274 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os + +kfp_version = os.environ["KFP_VERSION"] +disable_istio_sidecar = os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + + +class Controller(BaseHTTPRequestHandler): + def sync(self, parent, children): + # HACK: Currently using serving.kubeflow.org/inferenceservice to identify + # kubeflow user namespaces. + # TODO: let Kubeflow profile controller add a pipeline specific label to + # user namespaces and use that label instead. + pipeline_enabled = parent.get("metadata", {}).get( + "labels", {}).get("serving.kubeflow.org/inferenceservice") + + if not pipeline_enabled: + return {"status": {}, "children": []} + + # Compute status based on observed state. + desired_status = { + "kubeflow-pipelines-ready": \ + len(children["Secret.v1"]) == 1 and \ + len(children["ConfigMap.v1"]) == 1 and \ + len(children["Deployment.apps/v1"]) == 2 and \ + len(children["Service.v1"]) == 2 and \ + len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and \ + len(children["ServiceRole.rbac.istio.io/v1alpha1"]) == 1 and \ + len(children["ServiceRoleBinding.rbac.istio.io/v1alpha1"]) == 1 and \ + "True" or "False" + } + + # Generate the desired child object(s). + # parent is a namespace + namespace = parent.get("metadata", {}).get("name") + desired_resources = [ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": "bWluaW8=", # base64 for minio + "secretkey": "bWluaW8xMjM=", # base64 for minio123 + }, + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "metadata-grpc-configmap", + "namespace": namespace, + }, + "data": { + "METADATA_GRPC_SERVICE_HOST": + "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_PORT": "8080", + }, + }, + # Visualization server related manifests below + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app": "ml-pipeline-visualizationserver" + }, + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "selector": { + "matchLabels": { + "app": "ml-pipeline-visualizationserver" + }, + }, + "template": { + "metadata": { + "labels": { + "app": "ml-pipeline-visualizationserver" + }, + "annotations": disable_istio_sidecar and { + "sidecar.istio.io/inject": "false" + } or {}, + }, + "spec": { + "containers": [{ + "image": + "gcr.io/ml-pipeline/visualization-server:" + + kfp_version, + "imagePullPolicy": + "IfNotPresent", + "name": + "ml-pipeline-visualizationserver", + "ports": [{ + "containerPort": 8888 + }], + }], + "serviceAccountName": + "default-editor", + }, + }, + }, + }, + { + "apiVersion": "networking.istio.io/v1alpha3", + "kind": "DestinationRule", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "host": "ml-pipeline-visualizationserver", + "trafficPolicy": { + "tls": { + "mode": "ISTIO_MUTUAL" + } + } + } + }, + { + "apiVersion": "rbac.istio.io/v1alpha1", + "kind": "ServiceRole", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "rules": [{ + "services": ["ml-pipeline-visualizationserver.*"] + }] + } + }, + { + "apiVersion": "rbac.istio.io/v1alpha1", + "kind": "ServiceRoleBinding", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "subjects": [{ + "properties": { + "source.principal": + "cluster.local/ns/kubeflow/sa/ml-pipeline" + } + }], + "roleRef": { + "kind": "ServiceRole", + "name": "ml-pipeline-visualizationserver" + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ml-pipeline-visualizationserver", + "namespace": namespace, + }, + "spec": { + "ports": [{ + "name": "http", + "port": 8888, + "protocol": "TCP", + "targetPort": 8888, + }], + "selector": { + "app": "ml-pipeline-visualizationserver", + }, + }, + }, + # Artifact fetcher related resources below. + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app": "ml-pipeline-ui-artifact" + }, + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, + }, + "spec": { + "selector": { + "matchLabels": { + "app": "ml-pipeline-ui-artifact" + } + }, + "template": { + "metadata": { + "labels": { + "app": "ml-pipeline-ui-artifact" + }, + "annotations": disable_istio_sidecar and { + "sidecar.istio.io/inject": "false" + } or {}, + }, + "spec": { + "containers": [{ + "name": + "ml-pipeline-ui-artifact", + "image": + "gcr.io/ml-pipeline/frontend:" + kfp_version, + "imagePullPolicy": + "IfNotPresent", + "ports": [{ + "containerPort": 3000 + }] + }], + "serviceAccountName": + "default-editor" + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ml-pipeline-ui-artifact", + "namespace": namespace, + "labels": { + "app": "ml-pipeline-ui-artifact" + } + }, + "spec": { + "ports": [{ + "name": + "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000 + }], + "selector": { + "app": "ml-pipeline-ui-artifact" + } + } + }, + ] + print('Received request', parent, desired_resources) + + return {"status": desired_status, "children": desired_resources} + + def do_POST(self): + # Serve the sync() function as a JSON webhook. + observed = json.loads( + self.rfile.read(int(self.headers.get("content-length")))) + desired = self.sync(observed["parent"], observed["children"]) + + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write(bytes(json.dumps(desired), 'utf-8')) + + +HTTPServer(("", 80), Controller).serve_forever() diff --git a/pipeline/installs/multi-user/pipelines-ui/cluster-role-binding.yaml b/pipeline/installs/multi-user/pipelines-ui/cluster-role-binding.yaml new file mode 100644 index 00000000000..3539ff107e2 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-ui/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-ui +subjects: +- kind: ServiceAccount + name: ml-pipeline-ui diff --git a/pipeline/installs/multi-user/pipelines-ui/cluster-role.yaml b/pipeline/installs/multi-user/pipelines-ui/cluster-role.yaml new file mode 100644 index 00000000000..dc352c941f4 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-ui/cluster-role.yaml @@ -0,0 +1,42 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-ui +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - "kubeflow.org" + resources: + - viewers + verbs: + - create + - get + - list + - watch + - delete +- apiGroups: + - "argoproj.io" + resources: + - workflows + verbs: + - get + - list diff --git a/pipeline/installs/multi-user/pipelines-ui/configmap.yaml b/pipeline/installs/multi-user/pipelines-ui/configmap.yaml new file mode 100644 index 00000000000..11f1f551bd0 --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-ui/configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-ui-configmap +data: + # Temporary workarounds: + # 1. Using default-editor because default-viewer isn't bound to workload identity + viewer-pod-template.json: |- + { + "spec": { + "serviceAccountName": "default-editor" + } + } diff --git a/pipeline/installs/multi-user/pipelines-ui/deployment-patch.yaml b/pipeline/installs/multi-user/pipelines-ui/deployment-patch.yaml new file mode 100644 index 00000000000..3ff5950b75d --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-ui/deployment-patch.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline-ui +spec: + template: + spec: + volumes: + - name: config-volume + configMap: + name: ml-pipeline-ui-configmap + containers: + - name: ml-pipeline-ui + env: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: /etc/config/viewer-pod-template.json + - name: DEPLOYMENT + value: KUBEFLOW + - name: ARTIFACTS_SERVICE_PROXY_NAME + value: ml-pipeline-ui-artifact + - name: ARTIFACTS_SERVICE_PROXY_PORT + value: '80' + - name: ARTIFACTS_SERVICE_PROXY_ENABLED + value: 'true' + - name: ENABLE_AUTHZ + value: 'true' + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true diff --git a/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml b/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml new file mode 100644 index 00000000000..33ad4fc768b --- /dev/null +++ b/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +commonLabels: + app: ml-pipeline-ui +resources: +- cluster-role.yaml +- cluster-role-binding.yaml +- configmap.yaml diff --git a/pipeline/installs/multi-user/scheduled-workflow/cluster-role-binding.yaml b/pipeline/installs/multi-user/scheduled-workflow/cluster-role-binding.yaml new file mode 100644 index 00000000000..0495d0017a1 --- /dev/null +++ b/pipeline/installs/multi-user/scheduled-workflow/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-scheduledworkflow-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-scheduledworkflow-role +subjects: +- kind: ServiceAccount + name: ml-pipeline-scheduledworkflow diff --git a/pipeline/installs/multi-user/scheduled-workflow/cluster-role.yaml b/pipeline/installs/multi-user/scheduled-workflow/cluster-role.yaml new file mode 100644 index 00000000000..d08722c21e1 --- /dev/null +++ b/pipeline/installs/multi-user/scheduled-workflow/cluster-role.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-scheduledworkflow-role +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch diff --git a/pipeline/installs/multi-user/scheduled-workflow/deployment-patch.yaml b/pipeline/installs/multi-user/scheduled-workflow/deployment-patch.yaml new file mode 100644 index 00000000000..ea35690d816 --- /dev/null +++ b/pipeline/installs/multi-user/scheduled-workflow/deployment-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline-scheduledworkflow +spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: NAMESPACE + value: '' # Empty namespace let viewer controller watch all namespaces + valueFrom: null # HACK: https://github.com/kubernetes-sigs/kustomize/issues/2606 diff --git a/pipeline/installs/multi-user/scheduled-workflow/kustomization.yaml b/pipeline/installs/multi-user/scheduled-workflow/kustomization.yaml new file mode 100644 index 00000000000..ad2710f3363 --- /dev/null +++ b/pipeline/installs/multi-user/scheduled-workflow/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/pipeline/installs/multi-user/viewer-controller/cluster-role-binding.yaml b/pipeline/installs/multi-user/viewer-controller/cluster-role-binding.yaml new file mode 100644 index 00000000000..5e325bfc68e --- /dev/null +++ b/pipeline/installs/multi-user/viewer-controller/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ml-pipeline-viewer-crd-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ml-pipeline-viewer-controller-role +subjects: +- kind: ServiceAccount + name: ml-pipeline-viewer-crd-service-account diff --git a/pipeline/installs/multi-user/viewer-controller/cluster-role.yaml b/pipeline/installs/multi-user/viewer-controller/cluster-role.yaml new file mode 100644 index 00000000000..e2bca79710e --- /dev/null +++ b/pipeline/installs/multi-user/viewer-controller/cluster-role.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-viewer-controller-role +rules: +- apiGroups: + - '*' + resources: + - deployments + - services + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete diff --git a/pipeline/installs/multi-user/viewer-controller/deployment-patch.yaml b/pipeline/installs/multi-user/viewer-controller/deployment-patch.yaml new file mode 100644 index 00000000000..73e5d105063 --- /dev/null +++ b/pipeline/installs/multi-user/viewer-controller/deployment-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline-viewer-crd +spec: + template: + spec: + containers: + - name: ml-pipeline-viewer-crd + env: + - name: NAMESPACE + value: '' # Empty namespace let viewer controller watch all namespaces + valueFrom: null diff --git a/pipeline/installs/multi-user/viewer-controller/kustomization.yaml b/pipeline/installs/multi-user/viewer-controller/kustomization.yaml new file mode 100644 index 00000000000..b1f65469e1d --- /dev/null +++ b/pipeline/installs/multi-user/viewer-controller/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml