fix kustomize manifests for kubeflow (#1498)

* fix kustomize manifests for kubeflow

* fix standalone and external-db manifests

* remove old namespace file

* remove PV from kubeflow manifest

* fix katib-external-db reference outside of root

* fix katib-with-kubeflow-cert-manager

* Move image tags to katib-config.yaml and remove patches

* use common namespace kustomization

* Make kubeflow-cert use kubeflow as a base

* Remove katib-cert-generator job from kubeflow-cert-generator manifests

* Move pv-patch to patches folder

* Create katib-cert-manager and make kubeflowuse this as base

* Fix release and CI scripts for new layout

* Remove unnecessary cert-generator images from kustomization.yaml

* Remove unnecessary SA, CR and CRB from katib-cert-manager

* Remove commonLabel from katib-with-kubeflow

* Separate cert-generator from webhook kustomization

manifests/v1beta1/components/cert-generator/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- cert-generator.yaml
+- rbac.yaml

manifests/v1beta1/components/controller/katib-config.yaml

@@ -7,13 +7,13 @@ data:
   metrics-collector-sidecar: |-
     {
       "StdOut": {
-        "image": "docker.io/kubeflowkatib/file-metrics-collector"
+        "image": "docker.io/kubeflowkatib/file-metrics-collector:latest"
       },
       "File": {
-        "image": "docker.io/kubeflowkatib/file-metrics-collector"
+        "image": "docker.io/kubeflowkatib/file-metrics-collector:latest"
       },
       "TensorFlowEvent": {
-        "image": "docker.io/kubeflowkatib/tfevent-metrics-collector",
+        "image": "docker.io/kubeflowkatib/tfevent-metrics-collector:latest",
         "resources": {
           "limits": {
             "memory": "1Gi"
@@ -24,38 +24,38 @@ data:
   suggestion: |-
     {
       "random": {
-        "image": "docker.io/kubeflowkatib/suggestion-hyperopt"
+        "image": "docker.io/kubeflowkatib/suggestion-hyperopt:latest"
       },
       "tpe": {
-        "image": "docker.io/kubeflowkatib/suggestion-hyperopt"
+        "image": "docker.io/kubeflowkatib/suggestion-hyperopt:latest"
       },
       "grid": {
-        "image": "docker.io/kubeflowkatib/suggestion-chocolate"
+        "image": "docker.io/kubeflowkatib/suggestion-chocolate:latest"
       },
       "hyperband": {
-        "image": "docker.io/kubeflowkatib/suggestion-hyperband"
+        "image": "docker.io/kubeflowkatib/suggestion-hyperband:latest"
       },
       "bayesianoptimization": {
-        "image": "docker.io/kubeflowkatib/suggestion-skopt"
+        "image": "docker.io/kubeflowkatib/suggestion-skopt:latest"
       },
       "cmaes": {
-        "image": "docker.io/kubeflowkatib/suggestion-goptuna"
+        "image": "docker.io/kubeflowkatib/suggestion-goptuna:latest"
       },
       "enas": {
-        "image": "docker.io/kubeflowkatib/suggestion-enas",
+        "image": "docker.io/kubeflowkatib/suggestion-enas:latest",
         "resources": {
           "limits": {
             "memory": "200Mi"
           }
         }
       },
       "darts": {
-        "image": "docker.io/kubeflowkatib/suggestion-darts"
+        "image": "docker.io/kubeflowkatib/suggestion-darts:latest"
       }
     }
   early-stopping: |-
     {
       "medianstop": {
-        "image": "docker.io/kubeflowkatib/earlystopping-medianstop"
+        "image": "docker.io/kubeflowkatib/earlystopping-medianstop:latest"
       }
     }

manifests/v1beta1/components/controller/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- controller.yaml
+- katib-config.yaml
+- rbac.yaml
+- service.yaml
+- trial-templates.yaml

manifests/v1beta1/components/crd/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- experiment.yaml
+- suggestion.yaml
+- trial.yaml

manifests/v1beta1/components/db-manager/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- db-manager.yaml
+- service.yaml

manifests/v1beta1/components/mysql/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- mysql.yaml
+- pv.yaml
+- pvc.yaml
+- secret.yaml
+- service.yaml

manifests/v1beta1/components/namespace/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  # Namespace.
+  - namespace.yaml

manifests/v1beta1/components/ui/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- rbac.yaml
+- service.yaml
+- ui.yaml

manifests/v1beta1/components/webhook/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- webhooks.yaml

manifests/v1beta1/installs/katib-cert-manager/kustomization.yaml

@@ -0,0 +1,60 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  # Namespace.
+  - ../../components/namespace
+  # Katib controller.
+  - ../../components/controller/
+  # Katib CRDs.
+  - ../../components/crd/
+  # Katib DB manager.
+  - ../../components/db-manager/
+  # Katib DB mysql.
+  - ../../components/mysql/
+  # Katib UI.
+  - ../../components/ui/
+  # Katib webhooks.
+  - ../../components/webhook/
+  # Cert-manager certificate for webhooks
+  - certificate.yaml
+images:
+  - name: docker.io/kubeflowkatib/katib-controller
+    newName: docker.io/kubeflowkatib/katib-controller
+    newTag: latest
+  - name: docker.io/kubeflowkatib/katib-db-manager
+    newName: docker.io/kubeflowkatib/katib-db-manager
+    newTag: latest
+  - name: docker.io/kubeflowkatib/katib-ui
+    newName: docker.io/kubeflowkatib/katib-ui
+    newTag: latest
+
+patchesStrategicMerge:
+  - patches/katib-cert-injection.yaml
+
+vars:
+- fieldref:
+    fieldPath: metadata.namespace
+  name: KATIB_NAMESPACE
+  objref:
+    apiVersion: v1
+    kind: Service
+    name: katib-controller
+- fieldref:
+    fieldPath: metadata.name
+  name: KATIB_SERVICE_NAME
+  objref:
+    apiVersion: v1
+    kind: Service
+    name: katib-controller
+- name: KATIB_CERT_NAME
+  objref:
+      kind: Certificate
+      group: cert-manager.io
+      version: v1alpha2
+      name: katib-webhook-cert
+  fieldref:
+    fieldpath: metadata.name
+
+configurations:
+  - params.yaml

manifests/v1beta1/installs/katib-external-db/kustomization.yaml

@@ -3,43 +3,33 @@ kind: Kustomization
 namespace: kubeflow
 resources:
   # Namespace.
-  - ../../components/namespace.yaml
+  - ../../components/namespace/
   # Katib controller.
-  - ../../components/controller/controller.yaml
-  - ../../components/controller/service.yaml
-  - ../../components/controller/rbac.yaml
-  - ../../components/controller/katib-config.yaml
-  - ../../components/controller/trial-templates.yaml
+  - ../../components/controller/
   # Katib CRDs.
-  - ../../components/crd/experiment.yaml
-  - ../../components/crd/suggestion.yaml
-  - ../../components/crd/trial.yaml
+  - ../../components/crd/
   # Katib DB manager.
-  - ../../components/db-manager/db-manager.yaml
-  - ../../components/db-manager/service.yaml
+  - ../../components/db-manager/
   # Katib UI.
-  - ../../components/ui/ui.yaml
-  - ../../components/ui/service.yaml
-  - ../../components/ui/rbac.yaml
+  - ../../components/ui/
+  # Katib Cert Generator
+  - ../../components/cert-generator/
   # Katib webhooks.
-  - ../../components/webhook/cert-generator.yaml
-  - ../../components/webhook/rbac.yaml
-  - ../../components/webhook/webhooks.yaml
+  - ../../components/webhook/
 images:
   - name: docker.io/kubeflowkatib/katib-controller
-    newTag: latest
     newName: docker.io/kubeflowkatib/katib-controller
-  - name: docker.io/kubeflowkatib/katib-db-manager
     newTag: latest
+  - name: docker.io/kubeflowkatib/katib-db-manager
     newName: docker.io/kubeflowkatib/katib-db-manager
-  - name: docker.io/kubeflowkatib/katib-ui
     newTag: latest
+  - name: docker.io/kubeflowkatib/katib-ui
     newName: docker.io/kubeflowkatib/katib-ui
-  - name: docker.io/kubeflowkatib/cert-generator
     newTag: latest
+  - name: docker.io/kubeflowkatib/cert-generator
     newName: docker.io/kubeflowkatib/cert-generator
+    newTag: latest
 patchesStrategicMerge:
-  - ../katib-standalone/katib-config-patch.yaml
   - db-manager-patch.yaml
 # Modify katib-mysql-secrets with parameters for the DB.
 secretGenerator:

manifests/v1beta1/installs/katib-standalone/kustomization.yaml

@@ -3,46 +3,31 @@ kind: Kustomization
 namespace: kubeflow
 resources:
   # Namespace.
-  - ../../components/namespace.yaml
+  - ../../components/namespace/
   # Katib controller.
-  - ../../components/controller/controller.yaml
-  - ../../components/controller/service.yaml
-  - ../../components/controller/rbac.yaml
-  - ../../components/controller/katib-config.yaml
-  - ../../components/controller/trial-templates.yaml
+  - ../../components/controller/
   # Katib CRDs.
-  - ../../components/crd/experiment.yaml
-  - ../../components/crd/suggestion.yaml
-  - ../../components/crd/trial.yaml
+  - ../../components/crd/
   # Katib DB manager.
-  - ../../components/db-manager/db-manager.yaml
-  - ../../components/db-manager/service.yaml
+  - ../../components/db-manager/
   # Katib DB mysql.
-  - ../../components/mysql/mysql.yaml
-  - ../../components/mysql/service.yaml
-  - ../../components/mysql/pv.yaml
-  - ../../components/mysql/pvc.yaml
-  - ../../components/mysql/secret.yaml
+  - ../../components/mysql/
   # Katib UI.
-  - ../../components/ui/ui.yaml
-  - ../../components/ui/service.yaml
-  - ../../components/ui/rbac.yaml
+  - ../../components/ui/
+  # Katib Cert Generator
+  - ../../components/cert-generator/
   # Katib webhooks.
-  - ../../components/webhook/cert-generator.yaml
-  - ../../components/webhook/rbac.yaml
-  - ../../components/webhook/webhooks.yaml
+  - ../../components/webhook/
 images:
   - name: docker.io/kubeflowkatib/katib-controller
-    newTag: latest
     newName: docker.io/kubeflowkatib/katib-controller
-  - name: docker.io/kubeflowkatib/katib-db-manager
     newTag: latest
+  - name: docker.io/kubeflowkatib/katib-db-manager
     newName: docker.io/kubeflowkatib/katib-db-manager
-  - name: docker.io/kubeflowkatib/katib-ui
     newTag: latest
+  - name: docker.io/kubeflowkatib/katib-ui
     newName: docker.io/kubeflowkatib/katib-ui
-  - name: docker.io/kubeflowkatib/cert-generator
     newTag: latest
+  - name: docker.io/kubeflowkatib/cert-generator
     newName: docker.io/kubeflowkatib/cert-generator
-patchesStrategicMerge:
-  - katib-config-patch.yaml
+    newTag: latest