chore(deps): Update to Weaviate 1.25.1 (#2)

Signed-off-by: Thomas Vitale <ThomasVitale@users.noreply.github.com>
kadras-io · May 17, 2024 · a769505 · a769505
1 parent a2b8b4c
commit a769505
Show file tree

Hide file tree

Showing 20 changed files with 387 additions and 50 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -18,6 +18,6 @@ jobs:
       registry-server: ghcr.io
       registry-username: ${{ github.actor }}
       image: ${{ github.repository }}
-      version: 1.24.1
+      version: 1.25.1
     secrets:
       pull-request-token: ${{ secrets.GH_ORG_PAT }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ jobs:
     name: Integration Tests
     strategy:
       matrix:
-        k8s_version: [v1.27, v1.28, v1.29]
+        k8s_version: [v1.28, v1.29, v1.30]
     permissions:
       contents: read
     uses: kadras-io/github-reusable-workflows/.github/workflows/carvel-package-test-integration.yml@main

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-K8S_VERSION=v1.29
+K8S_VERSION=v1.30
 
 # Build package configuration
 build: package

diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@ A Carvel package for [Weaviate](https://weaviate.io), an AI-native vector databa
 
 ### Prerequisites
 
-* Kubernetes 1.27+
+* Kubernetes 1.28+
 * Carvel [`kctrl`](https://carvel.dev/kapp-controller/docs/latest/install/#installing-kapp-controller-cli-kctrl) CLI.
 * Carvel [kapp-controller](https://carvel.dev/kapp-controller) deployed in your Kubernetes cluster. You can install it with Carvel [`kapp`](https://carvel.dev/kapp/docs/latest/install) (recommended choice) or `kubectl`.
 

diff --git a/package/config/helm/upstream/weaviate/Chart.yaml b/package/config/helm/upstream/weaviate/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v2
-appVersion: 1.24.1
+appVersion: 1.25.0
 description: A Helm Chart to run Weaviate with all dependencies
 home: https://github.com/weaviate/weaviate
 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png
 name: weaviate
 type: application
-version: 16.8.4
+version: 17.0.0
diff --git a/package/config/helm/upstream/weaviate/templates/_helpers.tpl b/package/config/helm/upstream/weaviate/templates/_helpers.tpl
@@ -35,6 +35,15 @@
   {{- if (index .Values "modules" "generative-anyscale" "enabled") -}}
     {{ $modules = append $modules "generative-anyscale" }}
   {{- end -}}
+  {{- if (index .Values "modules" "generative-mistral" "enabled") -}}
+    {{ $modules = append $modules "generative-mistral" }}
+  {{- end -}}
+  {{- if (index .Values "modules" "generative-ollama" "enabled") -}}
+    {{ $modules = append $modules "generative-ollama" }}
+  {{- end -}}
+  {{- if (index .Values "modules" "generative-octoai" "enabled") -}}
+    {{ $modules = append $modules "generative-octoai" }}
+  {{- end -}}
   {{- if or (index .Values "modules" "img2vec-neural" "enabled") (index .Values "modules" "img2vec-neural" "inferenceUrl") -}}
     {{ $modules = append $modules "img2vec-neural" }}
   {{- end -}}
@@ -53,6 +62,9 @@
   {{- if or (index .Values "modules" "multi2vec-bind" "enabled") (index .Values "modules" "multi2vec-bind" "inferenceUrl") -}}
     {{ $modules = append $modules "multi2vec-bind" }}
   {{- end -}}
+  {{- if (index .Values "modules" "multi2vec-palm" "enabled") -}}
+    {{ $modules = append $modules "multi2vec-palm" }}
+  {{- end -}}
   {{- if (index .Values "modules" "text2vec-openai" "enabled") -}}
     {{ $modules = append $modules "text2vec-openai" }}
   {{- end -}}
@@ -71,6 +83,15 @@
   {{- if (index .Values "modules" "text2vec-aws" "enabled") -}}
     {{ $modules = append $modules "text2vec-aws" }}
   {{- end -}}
+  {{- if (index .Values "modules" "text2vec-voyageai" "enabled") -}}
+    {{ $modules = append $modules "text2vec-voyageai" }}
+  {{- end -}}
+  {{- if (index .Values "modules" "text2vec-ollama" "enabled") -}}
+    {{ $modules = append $modules "text2vec-ollama" }}
+  {{- end -}}
+  {{- if (index .Values "modules" "text2vec-octoai" "enabled") -}}
+    {{ $modules = append $modules "text2vec-octoai" }}
+  {{- end -}}
   {{- if (index .Values "modules" "ref2vec-centroid" "enabled") -}}
     {{ $modules = append $modules "ref2vec-centroid" }}
   {{- end -}}
@@ -80,6 +101,9 @@
   {{- if (index .Values "modules" "reranker-transformers" "enabled") -}}
     {{ $modules = append $modules "reranker-transformers" }}
   {{- end -}}
+  {{- if (index .Values "modules" "reranker-voyageai" "enabled") -}}
+    {{ $modules = append $modules "reranker-voyageai" }}
+  {{- end -}}
   {{- if (index .Values "backups" "filesystem" "enabled") -}}
     {{ $modules = append $modules "backup-filesystem" }}
   {{- end -}}
@@ -161,3 +185,54 @@ Usage:
     {{- printf "priorityClassName: %s" $priorityClassName -}}
   {{- end -}}
 {{- end -}}
+
+
+{{/*
+Raft cluster configuration settings
+*/}}
+{{- define "raft_configuration" -}}
+  {{- $replicas := .Values.replicas | int -}}
+  {{- $voters := .Values.env.RAFT_BOOTSTRAP_EXPECT | int -}}
+  {{- $metada_only_voters := false -}}
+  {{- if not (empty .Values.env.RAFT_METADATA_ONLY_VOTERS) -}}
+    {{- $metada_only_voters = .Values.env.RAFT_METADATA_ONLY_VOTERS -}}
+  {{- end -}} 
+  {{- if empty .Values.env.RAFT_BOOTSTRAP_EXPECT -}}
+    {{- if ge $replicas 10 -}}
+      {{- $voters = 5 -}}
+    {{- else if ge $replicas 3 -}}
+      {{- $voters = 3 -}}
+    {{- else -}}
+      {{- $voters = 1 -}}
+    {{- end -}}
+  {{- end -}}
+  {{- if gt $voters $replicas  -}}
+    {{- fail "env.RAFT_BOOTSTRAP_EXPECT value cannot be greater than replicas value" -}}
+  {{- end -}}
+  {{- if empty .Values.env.RAFT_JOIN -}}
+    {{- $nodes := list -}}
+    {{- range $i := until $voters -}}
+      {{- $node_name := list -}}
+      {{- $node_name = append $node_name "weaviate" -}}
+      {{- $node_name = append $node_name $i -}}
+      {{- $nodes = append $nodes (join "-" $node_name) -}}
+    {{- end -}}
+          - name: RAFT_JOIN
+            value: "{{ join "," $nodes }}"
+  {{- else -}}
+    {{- $votersCount := len (split "," .Values.env.RAFT_JOIN) -}}
+    {{- if empty .Values.env.RAFT_BOOTSTRAP_EXPECT }}
+      {{- $voters = $votersCount -}}
+    {{- end -}}
+    {{- if not (eq $votersCount $voters)  -}}
+      {{- fail "env.RAFT_BOOTSTRAP_EXPECT value needs to be equal to number of env.RAFT_JOIN nodes" -}}
+    {{- end -}}
+  {{- end -}}
+  {{- if empty .Values.env.RAFT_BOOTSTRAP_EXPECT }}
+          - name: RAFT_BOOTSTRAP_EXPECT
+            value: "{{ $voters }}"
+  {{- end -}}
+  {{- if and ($metada_only_voters) (le $replicas $voters) -}}
+    {{- fail "env.RAFT_METADATA_ONLY_VOTERS is true then .replicas size must be greater than env.RAFT_BOOTSTRAP_EXPECT" -}}
+  {{- end -}}
+{{- end -}}
diff --git a/package/config/helm/upstream/weaviate/templates/apiKeyMistralSecret.yaml b/package/config/helm/upstream/weaviate/templates/apiKeyMistralSecret.yaml
@@ -0,0 +1,12 @@
+{{ if and (index .Values "modules" "generative-mistral" "enabled") (index .Values "modules" "generative-mistral" "apiKey") }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: weaviate-mistral
+  labels:
+    app.kubernetes.io/name: weaviate
+    app.kubernetes.io/managed-by: helm
+type: Opaque
+data:
+  apiKey: {{ index .Values "modules" "generative-mistral" "apiKey" | b64enc }}
+{{ end }}
diff --git a/package/config/helm/upstream/weaviate/templates/apiKeyOctoAISecret.yaml b/package/config/helm/upstream/weaviate/templates/apiKeyOctoAISecret.yaml
@@ -0,0 +1,28 @@
+{{-  $t2vOctoAI := and (index .Values "modules" "text2vec-octoai" "enabled") (index .Values "modules" "text2vec-octoai" "apiKey") -}}
+{{-  $generativeOctoAI := and (index .Values "modules" "generative-octoai" "enabled") (index .Values "modules" "generative-octoai" "apiKey") }}
+{{-  $octoai := or ($t2vOctoAI) ($generativeOctoAI) }}
+{{- if $octoai }}
+
+{{- $apiKeys := list }}
+{{- if $t2vOctoAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "text2vec-octoai" "apiKey") -}}
+{{- end }}
+{{- if $generativeOctoAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "generative-octoai" "apiKey") -}}
+{{- end }}
+
+{{- if gt (len ($apiKeys | uniq)) 1 -}}
+  {{- fail "OctoAI modules activated, but their keys differ. Use the same key on all activated OctoAI modules." }}
+{{- end }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: weaviate-octoai
+  labels:
+    app.kubernetes.io/name: weaviate
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+type: Opaque
+data:
+  apiKey: {{ index $apiKeys 0 | b64enc }}
+{{ end }}
diff --git a/package/config/helm/upstream/weaviate/templates/apiKeyPaLMSecret.yaml b/package/config/helm/upstream/weaviate/templates/apiKeyPaLMSecret.yaml
@@ -1,6 +1,7 @@
 {{-  $t2vPaLM := and (index .Values "modules" "text2vec-palm" "enabled") (index .Values "modules" "text2vec-palm" "apiKey") -}}
 {{-  $generativePaLM := and (index .Values "modules" "generative-palm" "enabled") (index .Values "modules" "generative-palm" "apiKey") }}
-{{-  $palm := or ($t2vPaLM) ($generativePaLM) }}
+{{-  $m2vPaLM := and (index .Values "modules" "multi2vec-palm" "enabled") (index .Values "modules" "multi2vec-palm" "apiKey") -}}
+{{-  $palm := or ($t2vPaLM) ($generativePaLM) ($m2vPaLM) }}
 {{- if $palm }}
 
 {{- $apiKeys := list }}
@@ -10,6 +11,9 @@
 {{- if $generativePaLM }}
   {{- $apiKeys = append $apiKeys (index .Values "modules" "generative-palm" "apiKey") -}}
 {{- end }}
+{{- if $m2vPaLM }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-palm" "apiKey") -}}
+{{- end }}
 
 {{- if gt (len ($apiKeys | uniq)) 1 -}}
   {{- fail "Google PaLM modules activated, but their keys differ. Use the same key on all activated Google PaLM modules." }}
@@ -26,6 +30,8 @@ type: Opaque
 data:
   {{- if (index .Values "modules" "generative-palm" "apiKey") }}
   apiKey: {{ index .Values "modules" "generative-palm" "apiKey" | b64enc }}
+  {{- else if (index .Values "modules" "multi2vec-palm" "apiKey") }}
+  apiKey: {{ index .Values "modules" "multi2vec-palm" "apiKey" | b64enc }}
   {{- else }}
   apiKey: {{ index .Values "modules" "text2vec-palm" "apiKey" | b64enc }}
   {{- end }}

diff --git a/package/config/helm/upstream/weaviate/templates/apiKeyVoyageAISecret.yaml b/package/config/helm/upstream/weaviate/templates/apiKeyVoyageAISecret.yaml
@@ -0,0 +1,32 @@
+{{-  $t2vVoyageAI := and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") -}}
+{{-  $rerankerVoyageAI := and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey") }}
+{{-  $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) }}
+{{- if $voyageai }}
+
+{{- $apiKeys := list }}
+{{- if $t2vVoyageAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "text2vec-voyageai" "apiKey") -}}
+{{- end }}
+{{- if $rerankerVoyageAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-voyageai" "apiKey") -}}
+{{- end }}
+
+{{- if gt (len ($apiKeys | uniq)) 1 -}}
+  {{- fail "VoyageAI modules activated, but their keys differ. Use the same key on all activated VoyageAI modules." }}
+{{- end }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: weaviate-voyageai
+  labels:
+    app.kubernetes.io/name: weaviate
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+type: Opaque
+data:
+  {{- if (index .Values "modules" "reranker-voyageai" "apiKey") }}
+  apiKey: {{ index .Values "modules" "reranker-voyageai" "apiKey" | b64enc }}
+  {{- else }}
+  apiKey: {{ index .Values "modules" "text2vec-voyageai" "apiKey" | b64enc }}
+  {{- end }}
+{{ end }}
diff --git a/package/config/helm/upstream/weaviate/templates/awsSecret.yaml b/package/config/helm/upstream/weaviate/templates/awsSecret.yaml
@@ -18,7 +18,7 @@
   {{- fail "AWS modules activated, but with both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. Please choose one configuration to pass AWS credentials." }}
 {{- end }}
 
-{{- if not $backupS3 -}}
+{{- if ($awsSecrets) -}}
 apiVersion: v1
 kind: Secret
 metadata:

diff --git a/package/config/helm/upstream/weaviate/templates/weaviateStatefulset.yaml b/package/config/helm/upstream/weaviate/templates/weaviateStatefulset.yaml
@@ -9,6 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
   replicas: {{ .Values.replicas }}
+  podManagementPolicy: {{ .Values.podManagementPolicy }}
   updateStrategy:
 {{ toYaml .Values.updateStrategy | indent 4}}
   serviceName: {{ .Values.service.name }}-headless
@@ -98,6 +99,7 @@ spec:
           - name: DEFAULT_VECTORIZER_MODULE
             value: {{ index .Values "modules" "default_vectorizer_module" | trim }}
           {{ template "enabled_modules" . }}
+          {{ template "raft_configuration" . }}
           {{- if index .Values "modules" "text2vec-transformers" "enabled" }}
           - name: TRANSFORMERS_INFERENCE_API
             value: http://{{ index .Values "modules" "text2vec-transformers" "fullnameOverride" }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:8080
@@ -211,7 +213,7 @@ spec:
                 name: weaviate-cohere
                 key: apiKey
           {{- end  }}
-          {{- if or (and (index .Values "modules" "text2vec-palm" "enabled") (index .Values "modules" "text2vec-palm" "apiKey")) (and (index .Values "modules" "generative-palm" "enabled") (index .Values "modules" "generative-palm" "apiKey")) }}
+          {{- if or (and (index .Values "modules" "text2vec-palm" "enabled") (index .Values "modules" "text2vec-palm" "apiKey")) (and (index .Values "modules" "generative-palm" "enabled") (index .Values "modules" "generative-palm" "apiKey")) (and (index .Values "modules" "multi2vec-palm" "enabled") (index .Values "modules" "multi2vec-palm" "apiKey")) }}
           - name: PALM_APIKEY
             valueFrom:
               secretKeyRef:
@@ -225,13 +227,34 @@ spec:
                 name: weaviate-jinaai
                 key: apiKey
           {{- end  }}
+          {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) }}
+          - name: VOYAGEAI_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-voyageai
+                key: apiKey
+          {{- end  }}
           {{- if and (index .Values "modules" "generative-anyscale" "enabled") (index .Values "modules" "generative-anyscale" "apiKey") }}
           - name: ANYSCALE_APIKEY
             valueFrom:
               secretKeyRef:
                 name: weaviate-anyscale
                 key: apiKey
           {{- end  }}
+          {{- if and (index .Values "modules" "generative-mistral" "enabled") (index .Values "modules" "generative-mistral" "apiKey") }}
+          - name: MISTRAL_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-mistral
+                key: apiKey
+          {{- end  }}
+          {{- if or (and (index .Values "modules" "text2vec-octoai" "enabled") (index .Values "modules" "text2vec-octoai" "apiKey")) (and (index .Values "modules" "generative-octoai" "enabled") (index .Values "modules" "generative-octoai" "apiKey")) }}
+          - name: OCTOAI_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-octoai
+                key: apiKey
+          {{- end  }}
           {{- if and (index .Values "backups" "filesystem" "enabled") (index .Values "backups" "filesystem" "envconfig") }}
             {{- range $key, $value := index .Values "backups" "filesystem" "envconfig" }}
           - name: {{ $key }}
@@ -384,28 +407,31 @@ spec:
           {{- end }}
         {{- if .Values.startupProbe.enabled }}
         startupProbe:
-          httpGet:
-            path: /v1/.well-known/ready
-            port: 8080
+          {{- $probeType := .Values.startupProbe.probeType }}
+          {{- $probe := index .Values.startupProbe.probe $probeType }}
+          {{ $probeType }}:
+            {{- toYaml $probe | nindent 12 }}
           initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.startupProbe.periodSeconds }}
           failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           successThreshold: {{ .Values.startupProbe.successThreshold }}
           timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
         {{- end }}
         livenessProbe:
-          httpGet:
-            path: /v1/.well-known/live
-            port: 8080
+          {{- $probeType := .Values.livenessProbe.probeType }}
+          {{- $probe := index .Values.livenessProbe.probe $probeType }}
+          {{ $probeType }}:
+            {{- toYaml $probe | nindent 12 }}
           initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
           failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           successThreshold: {{ .Values.livenessProbe.successThreshold }}
           timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
         readinessProbe:
-          httpGet:
-            path: /v1/.well-known/ready
-            port: 8080
+          {{- $probeType := .Values.readinessProbe.probeType }}
+          {{- $probe := index .Values.readinessProbe.probe $probeType }}
+          {{ $probeType }}:
+            {{- toYaml $probe | nindent 12 }}
           initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
           failureThreshold: {{ .Values.readinessProbe.failureThreshold }}