feat(artifact): add minio and milvus for artifact storage (#999)

Because

- To store embedding vectors & text content as knowledge base
- To store user-uploaded files (pdf, txt, markdown, jpg, png)

This commit

- add minio and milvus services in compose and helm deployment

resolves ins-4119

.env

@@ -177,3 +177,21 @@ REGISTRY_VERSION=2.8.3
 REGISTRY_HOST=registry
 REGISTRY_PORT=5000
 REGISTRY_EXTERNAL_PORT=5001
+
+# milvus
+MILVUS_IMAGE=milvusdb/milvus
+MILVUS_VERSION=v2.4.1
+MILVUS_HOST=milvus
+MILVUS_DATA_DIR=/var/lib/milvus/etcd
+MILVUS_PORT=19530
+MILVUS_METRIC_PORT=9091
+
+# minio
+MINIO_IMAGE=quay.io/minio/minio
+MINIO_VERSION=RELEASE.2024-05-10T01-41-38Z
+MINIO_HOST=minio
+MINIO_PORT=9000
+MINIO_EXTERNAL_PORT=19000
+MINIO_CONSOLE_PORT=9001
+MINIO_CONSOLE_EXTERNAL_PORT=19001
+MINIO_DATA_DIR=~/.local/instill/minio/data

.github/workflows/helm-integration-test-backend.yml

@@ -79,6 +79,9 @@ jobs:
             --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \
             --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \
             --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \
+            --set milvus.broker.resources.requests.cpu=0.25 \
+            --set milvus.autorecovery.resources.requests.cpu=0.25 \
+            --set milvus.proxy.resources.requests.cpu=0.25 \
             --set tags.observability=false \
             --set tags.prometheusStack=false
 
@@ -189,6 +192,9 @@ jobs:
             --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \
             --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \
             --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \
+            --set milvus.broker.resources.requests.cpu=0.25 \
+            --set milvus.autorecovery.resources.requests.cpu=0.25 \
+            --set milvus.proxy.resources.requests.cpu=0.25 \
             --set tags.observability=false \
             --set tags.prometheusStack=false
 

.github/workflows/helm-integration-test-console.yml

@@ -68,6 +68,9 @@ jobs:
             --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \
             --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \
             --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \
+            --set milvus.broker.resources.requests.cpu=0.25 \
+            --set milvus.autorecovery.resources.requests.cpu=0.25 \
+            --set milvus.proxy.resources.requests.cpu=0.25 \
             --set tags.observability=false \
             --set tags.prometheusStack=false
 
@@ -201,6 +204,9 @@ jobs:
             --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \
             --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \
             --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \
+            --set milvus.broker.resources.requests.cpu=0.25 \
+            --set milvus.autorecovery.resources.requests.cpu=0.25 \
+            --set milvus.proxy.resources.requests.cpu=0.25 \
             --set tags.observability=false \
             --set tags.prometheusStack=false
 

charts/core/Chart.lock

@@ -1,4 +1,7 @@
 dependencies:
+- name: milvus
+  repository: https://zilliztech.github.io/milvus-helm/
+  version: 4.1.30
 - name: kuberay-operator
   repository: https://ray-project.github.io/kuberay-helm/
   version: 1.1.1
@@ -20,5 +23,5 @@ dependencies:
 - name: grafana
   repository: https://grafana.github.io/helm-charts
   version: 7.0.19
-digest: sha256:54af8013247538db390dec1acf2dfb3b7b0d70c039e1d57763d689b6f48aea8e
-generated: "2024-05-12T04:49:12.15233+08:00"
+digest: sha256:12d974d5198de61d676406882a96f0d7e589bc4eef28e2ae4c303a5b1d787b26
+generated: "2024-05-21T00:12:29.456857+08:00"

charts/core/Chart.yaml

@@ -18,6 +18,9 @@ maintainers:
   - name: Instill AI LTD
     url: https://github.com/instill-ai
 dependencies:
+  - name: milvus
+    repository: https://zilliztech.github.io/milvus-helm/
+    version: 4.1.30
   - name: kuberay-operator
     repository: https://ray-project.github.io/kuberay-helm/
     version: 1.1.1

charts/core/README.md

@@ -8,14 +8,14 @@ The Helm chart of Instill Core
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.bitnami.com/bitnami | etcd | 8.8.1 |
 | https://grafana.github.io/helm-charts | grafana | 7.0.19 |
 | https://helm.elastic.co | elasticsearch | 7.17.3 |
 | https://helm.influxdata.com | influxdb2 | 2.1.1 |
 | https://jaegertracing.github.io/helm-charts | jaeger | 0.71.2 |
 | https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.59.1 |
 | https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 48.2.1 |
-| https://ray-project.github.io/kuberay-helm/ | kuberay-operator | 1.0.0 |
+| https://ray-project.github.io/kuberay-helm | kuberay-operator | 1.1.1 |
+| https://zilliztech.github.io/milvus-helm | milvus | 4.1.30 |
 
 ## Install
 

charts/core/templates/_helpers.tpl

@@ -389,6 +389,21 @@ registry
   {{- printf "5001" -}}
 {{- end -}}
 
+{{/*
+milvus
+*/}}
+{{- define "core.milvus" -}}
+  {{- printf "%s-milvus" (include "core.fullname" .) -}}
+{{- end -}}
+
+{{- define "core.milvus.port" -}}
+  {{- printf "19530" -}}
+{{- end -}}
+
+{{- define "core.milvus.metricPort" -}}
+  {{- printf "9091" -}}
+{{- end -}}
+
 {{/*
 internal TLS secret names
 */}}

charts/core/values.yaml

@@ -674,11 +674,11 @@ rayService:
   headGroupSpec:
     resources:
       limits:
-        cpu: "2"
+        cpu: "1"
         memory: "4Gi"
         nvidia.com/gpu: 0
       requests:
-        cpu: "2"
+        cpu: "1"
         memory: "4Gi"
         nvidia.com/gpu: 0
     affinity: {}
@@ -1471,6 +1471,97 @@ registry:
         idletimeout: 300s
       tls:
         enabled: false
+# -- The configuration of milvus
+# https://github.com/zilliztech/milvus-helm/blob/master/charts/milvus/values.yaml
+milvus:
+  image:
+    all:
+      repository: milvusdb/milvus
+      tag: v2.4.1
+      pullPolicy: IfNotPresent
+  nodeselector: {}
+  tolerations: []
+  affinity: {}
+  log:
+    level: "info"
+    file:
+      maxSize: 10   # MB
+      maxAge: 12    # day
+      maxBackups: 2
+    format: "json"  # text/json
+  cluster:
+    enabled: true
+  rootCoordinator:
+    enabled: false
+  queryCoordinator:
+    enabled: false
+  indexCoordinator:
+    enabled: false
+  dataCoordinator:
+    enabled: false
+  mixCoordinator:
+    enabled: true
+  minio:
+    mode: standalone
+    replicas: 1
+    persistence:
+      size: 50Gi
+  etcd:
+    enabled: true
+    replicaCount: 1
+  pulsar:
+    enabled: false
+    bookkeeper:
+      replicaCount: 1
+      volumes:
+        journal:
+          name: journal
+          size: 10Gi
+        ledgers:
+          name: ledgers
+          size: 20Gi
+      resources:
+        requests:
+          memory:
+          cpu: 0.5
+  kafka:
+    enabled: true
+    replicaCount: 1
+    defaultReplicationFactor: 1
+    offsetsTopicReplicationFactor: 1
+    persistence:
+      enabled: true
+      storageClass:
+      accessMode: ReadWriteOnce
+      size: 30Gi
+    zookeeper:
+      enabled: true
+      replicaCount: 1
+  broker:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: 0.5
+  autorecovery:
+    resources:
+      requests:
+        cpu: 0.5
+  proxy:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: 0.5
+  # https://github.com/milvus-io/milvus/blob/master/configs/milvus.yaml
+  extraConfigFiles:
+    user.yaml: |+
+      etcd:
+        rootPath: core-milvus
+        log:
+          level: info
+      minio:
+        bucketName: core-milvus
+      pulsar:
+        namespace: instill-ai
 tags:
   model: true
   observability: true

docker-compose-latest.yml

@@ -166,3 +166,13 @@ services:
   registry:
     ports:
       - ${REGISTRY_EXTERNAL_PORT}:5000
+
+  milvus:
+    ports:
+      - ${MILVUS_PORT}:19530
+    command: milvus run standalone
+
+  minio:
+    ports:
+      - ${MINIO_EXTERNAL_PORT}:${MINIO_PORT}
+      - ${MINIO_CONSOLE_EXTERNAL_PORT}:${MINIO_CONSOLE_PORT}