Katib: Re-add per component packages (kubeflow#1163)

This was reverted in a03973b due to test failures. This re-introduces the packages with the issues fixed. See kubeflow#1141 for more context.
surajkota · May 13, 2020 · 8061380 · 8061380
1 parent 3f961d5
commit 8061380
Show file tree

Hide file tree

Showing 81 changed files with 2,276 additions and 0 deletions.
diff --git a/hack/generate_tests.py b/hack/generate_tests.py
@@ -18,6 +18,7 @@
   # TODO(https://github.com/kubeflow/manifests/issues/1052): Remove this
   # after the move to v3 is done.
   "tests/legacy_kustomizations",
+  "katib/installs",
   ]
 
 # The subdirectory to story the expected manifests in

diff --git a/katib/components/katib-controller/kustomization.yaml b/katib/components/katib-controller/kustomization.yaml
@@ -0,0 +1,27 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../katib-controller/base/katib-configmap.yaml
+  - ../../katib-controller/base/katib-controller-deployment.yaml
+  - ../../katib-controller/base/katib-controller-rbac.yaml
+  - ../../katib-controller/base/katib-controller-secret.yaml
+  - ../../katib-controller/base/katib-controller-service.yaml
+  - ../../katib-controller/base/katib-ui-deployment.yaml
+  - ../../katib-controller/base/katib-ui-rbac.yaml
+  - ../../katib-controller/base/katib-ui-service.yaml
+  - ../../katib-controller/base/trial-template-configmap.yaml
+  - ../../katib-controller/overlays/application/application.yaml
+  - ../../katib-controller/overlays/istio/katib-ui-virtual-service.yaml
+images:
+  - name: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-controller
+    newTag: v0.8.0
+    newName: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-controller
+  - name: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-ui
+    newTag: v0.8.0
+    newName: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-ui
+commonLabels:
+  app.kubernetes.io/component: katib
+  app.kubernetes.io/name: katib-controller
+configurations:
+  - ../../katib-controller/overlays/istio/params.yaml
diff --git a/katib/components/katib-db-manager/kustomization.yaml b/katib/components/katib-db-manager/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../katib-controller/base/katib-db-manager-deployment.yaml
+  - ../../katib-controller/base/katib-db-manager-service.yaml
+images:
+  - name: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-db-manager
+    newTag: v0.8.0
+    newName: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-db-manager
diff --git a/katib/components/katib-db-mysql/kustomization.yaml b/katib/components/katib-db-mysql/kustomization.yaml
@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../katib-controller/base/katib-mysql-deployment.yaml
+  - ../../katib-controller/base/katib-mysql-pvc.yaml
+  - ../../katib-controller/base/katib-mysql-secret.yaml
+  - ../../katib-controller/base/katib-mysql-service.yaml
+images:
+  - name: mysql
+    newTag: "8"
+    newName: mysql
diff --git a/katib/installs/katib-external-db/katib-db-manager-deployment.yaml b/katib/installs/katib-external-db/katib-db-manager-deployment.yaml
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: katib-db-manager
+spec:
+  template:
+    spec:
+      containers:
+      - name: katib-db-manager
+        env:
+          - name: DB_NAME
+            value: mysql
+          - name: DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: katib-mysql-secrets
+                key: DB_USER
+          - name: DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: katib-mysql-secrets
+                key: DB_PASSWORD
+          - name: KATIB_MYSQL_DB_DATABASE
+            valueFrom:
+              secretKeyRef:
+                name: katib-mysql-secrets
+                key: KATIB_MYSQL_DB_DATABASE
+          - name: KATIB_MYSQL_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: katib-mysql-secrets
+                key: KATIB_MYSQL_DB_HOST
+          - name: KATIB_MYSQL_DB_PORT
+            valueFrom:
+              secretKeyRef:
+                name: katib-mysql-secrets
+                key: KATIB_MYSQL_DB_PORT
diff --git a/katib/installs/katib-external-db/kustomization.yaml b/katib/installs/katib-external-db/kustomization.yaml
@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../components/katib-controller
+  - ../../components/katib-db-manager
+patchesStrategicMerge:
+  - katib-db-manager-deployment.yaml
+secretGenerator:
+- name: katib-mysql-secrets
+  env: secrets.env
+commonLabels:
+  app.kubernetes.io/component: katib
+  app.kubernetes.io/name: katib-controller
diff --git a/katib/installs/katib-external-db/secrets.env b/katib/installs/katib-external-db/secrets.env
@@ -0,0 +1,5 @@
+KATIB_MYSQL_DB_DATABASE=
+KATIB_MYSQL_DB_HOST=
+KATIB_MYSQL_DB_PORT=
+DB_USER=
+DB_PASSWORD=
diff --git a/katib/installs/katib-standalone-ibm/kustomization.yaml b/katib/installs/katib-standalone-ibm/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../katib-standalone
+patchesStrategicMerge:
+- ../../katib-controller/overlays/ibm-storage-config/katib-mysql-deployment.yaml
+images:
+  - name: mysql
+    newTag: "5.6"
+    newName: mysql
diff --git a/katib/installs/katib-standalone/kustomization.yaml b/katib/installs/katib-standalone/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+resources:
+  - ../../components/katib-controller
+  - ../../components/katib-db-manager
+  - ../../components/katib-db-mysql
+commonLabels:
+  app.kubernetes.io/component: katib
+  app.kubernetes.io/name: katib-controller
diff --git a/tests/katib/installs/katib-external-db/kustomize_test.go b/tests/katib/installs/katib-external-db/kustomize_test.go
@@ -0,0 +1,15 @@
+package katib_external_db
+
+import (
+	"github.com/kubeflow/manifests/tests"
+	"testing"
+)
+
+func TestKustomize(t *testing.T) {
+	testCase := &tests.KustomizeTestCase{
+		Package:  "../../../../katib/installs/katib-external-db",
+		Expected: "test_data/expected",
+	}
+
+	tests.RunTestCase(t, testCase)
+}
diff --git a/...katib-external-db/test_data/expected/app.k8s.io_v1beta1_application_katib-controller.yaml b/...katib-external-db/test_data/expected/app.k8s.io_v1beta1_application_katib-controller.yaml
@@ -0,0 +1,70 @@
+apiVersion: app.k8s.io/v1beta1
+kind: Application
+metadata:
+  labels:
+    app.kubernetes.io/component: katib
+    app.kubernetes.io/name: katib-controller
+  name: katib-controller
+  namespace: kubeflow
+spec:
+  addOwnerRef: true
+  componentKinds:
+  - group: core
+    kind: Service
+  - group: apps
+    kind: Deployment
+  - group: core
+    kind: Secret
+  - group: core
+    kind: ServiceAccount
+  - group: kubeflow.org
+    kind: Experiment
+  - group: kubeflow.org
+    kind: Suggestion
+  - group: kubeflow.org
+    kind: Trial
+  descriptor:
+    description: Katib is a service for hyperparameter tuning and neural architecture
+      search.
+    keywords:
+    - katib
+    - katib-controller
+    - hyperparameter tuning
+    links:
+    - description: About
+      url: https://github.com/kubeflow/katib
+    maintainers:
+    - email: gaoce@caicloud.io
+      name: Ce Gao
+    - email: johnugeo@cisco.com
+      name: Johnu George
+    - email: liuhougang6@126.com
+      name: Hougang Liu
+    - email: ricliu@google.com
+      name: Richard Liu
+    - email: yuji.oshima0x3fd@gmail.com
+      name: YujiOshima
+    - email: andrey.velichkevich@gmail.com
+      name: Andrey Velichkevich
+    owners:
+    - email: gaoce@caicloud.io
+      name: Ce Gao
+    - email: johnugeo@cisco.com
+      name: Johnu George
+    - email: liuhougang6@126.com
+      name: Hougang Liu
+    - email: ricliu@google.com
+      name: Richard Liu
+    - email: yuji.oshima0x3fd@gmail.com
+      name: YujiOshima
+    - email: andrey.velichkevich@gmail.com
+      name: Andrey Velichkevich
+    type: katib
+    version: v1alpha3
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: katib
+      app.kubernetes.io/instance: katib-controller
+      app.kubernetes.io/managed-by: kfctl
+      app.kubernetes.io/name: katib-controller
+      app.kubernetes.io/part-of: kubeflow
diff --git a/...ib/installs/katib-external-db/test_data/expected/apps_v1_deployment_katib-controller.yaml b/...ib/installs/katib-external-db/test_data/expected/apps_v1_deployment_katib-controller.yaml
@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: katib-controller
+    app.kubernetes.io/component: katib
+    app.kubernetes.io/name: katib-controller
+  name: katib-controller
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: katib-controller
+      app.kubernetes.io/component: katib
+      app.kubernetes.io/name: katib-controller
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        sidecar.istio.io/inject: "false"
+      labels:
+        app: katib-controller
+        app.kubernetes.io/component: katib
+        app.kubernetes.io/name: katib-controller
+    spec:
+      containers:
+      - args:
+        - --webhook-port=8443
+        command:
+        - ./katib-controller
+        env:
+        - name: KATIB_CORE_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-controller:v0.8.0
+        imagePullPolicy: IfNotPresent
+        name: katib-controller
+        ports:
+        - containerPort: 8443
+          name: webhook
+          protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/cert
+          name: cert
+          readOnly: true
+      serviceAccountName: katib-controller
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: katib-controller
diff --git a/...ib/installs/katib-external-db/test_data/expected/apps_v1_deployment_katib-db-manager.yaml b/...ib/installs/katib-external-db/test_data/expected/apps_v1_deployment_katib-db-manager.yaml
@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: katib
+    app.kubernetes.io/component: katib
+    app.kubernetes.io/name: katib-controller
+    component: db-manager
+  name: katib-db-manager
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: katib
+      app.kubernetes.io/component: katib
+      app.kubernetes.io/name: katib-controller
+      component: db-manager
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
+      labels:
+        app: katib
+        app.kubernetes.io/component: katib
+        app.kubernetes.io/name: katib-controller
+        component: db-manager
+      name: katib-db-manager
+    spec:
+      containers:
+      - command:
+        - ./katib-db-manager
+        env:
+        - name: DB_NAME
+          value: mysql
+        - name: DB_USER
+          valueFrom:
+            secretKeyRef:
+              key: DB_USER
+              name: katib-mysql-secrets-kmcg6hfkfg
+        - name: DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: DB_PASSWORD
+              name: katib-mysql-secrets-kmcg6hfkfg
+        - name: KATIB_MYSQL_DB_DATABASE
+          valueFrom:
+            secretKeyRef:
+              key: KATIB_MYSQL_DB_DATABASE
+              name: katib-mysql-secrets-kmcg6hfkfg
+        - name: KATIB_MYSQL_DB_HOST
+          valueFrom:
+            secretKeyRef:
+              key: KATIB_MYSQL_DB_HOST
+              name: katib-mysql-secrets-kmcg6hfkfg
+        - name: KATIB_MYSQL_DB_PORT
+          valueFrom:
+            secretKeyRef:
+              key: KATIB_MYSQL_DB_PORT
+              name: katib-mysql-secrets-kmcg6hfkfg
+        image: gcr.io/kubeflow-images-public/katib/v1alpha3/katib-db-manager:v0.8.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /bin/grpc_health_probe
+            - -addr=:6789
+          failureThreshold: 5
+          initialDelaySeconds: 10
+          periodSeconds: 60
+        name: katib-db-manager
+        ports:
+        - containerPort: 6789
+          name: api
+        readinessProbe:
+          exec:
+            command:
+            - /bin/grpc_health_probe
+            - -addr=:6789
+          initialDelaySeconds: 5