Add deployment of groot (#1858)

* Fix a bug when finding frontend port in multiple node cluster * Update global parameters in charts, compatibility of JDK 9+ in data-load-tools * Several schema enhancements.
alibaba · Aug 23, 2022 · 23dcae0 · 23dcae0
1 parent 4aac5b5
commit 23dcae0
Show file tree

Hide file tree

Showing 31 changed files with 257 additions and 165 deletions.
diff --git a/charts/graphscope-store/README.md b/charts/graphscope-store/README.md
@@ -87,6 +87,8 @@ Here we give a list of most frequently used parameters.
 | commonAnnotations | Annotations to add to all deployed objects | {} |
 | executor | Executor type, "maxgraph" or "gaia" | maxgraph |
 | javaOpts | Java options | "" |
+| auth.username | Username | "" |
+| auth.password | Password | "" |
 
 
 ### Statefulset parameters
@@ -127,11 +129,11 @@ $ helm install my-release \
 ```
 
 Add multiple extra config to the component which is defined in the configmap by
-`--set extraConfig=k1=v1:k2=v2`. Note we use `:` to seperate config items. For example,
+`--set extraConfig=k1=v1;k2=v2`. Note we use `;` to seperate config items. For example,
 
 ```bash
 $ helm install my-release \
-  --set extraConfig=k1=v1:k2=v2 graphscope/graphscope-store
+  --set extraConfig=k1=v1;k2=v2 graphscope/graphscope-store
 ```
 
 

diff --git a/charts/graphscope-store/templates/NOTES.txt b/charts/graphscope-store/templates/NOTES.txt
@@ -1,9 +1,9 @@
 1. Get the application URL by running these commands:
 
 {{- if contains "NodePort" .Values.frontend.service.type }}
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  export GRPC_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "graphscope-store.frontend.fullname" . }})
-  export GREMLIN_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[1].nodePort}" services {{ include "graphscope-store.frontend.fullname" . }})
+  export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get pod {{ include "graphscope-store.frontend.fullname" . }}-0  -o jsonpath="{.status.hostIP}")
+  export GRPC_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[0].nodePort}")
+  export GREMLIN_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[1].nodePort}")
   echo "GRPC endpoint is: ${NODE_IP}:${GRPC_PORT}"
   echo "GREMLIN endpoint is: ${NODE_IP}:${GREMLIN_PORT}"
 {{- end }}
diff --git a/charts/graphscope-store/templates/_helpers.tpl b/charts/graphscope-store/templates/_helpers.tpl
@@ -70,7 +70,27 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 Return the proper graphscope-store image name
 */}}
 {{- define "graphscope-store.image" -}}
-{{ include "graphscope-store.images.image" . }}
+{{ include "graphscope-store.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper image name
+{{ include "graphscope-store.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
+*/}}
+{{- define "graphscope-store.images.image" -}}
+{{- $registryName := .imageRoot.registry -}}
+{{- $repositoryName := .imageRoot.repository -}}
+{{- $tag := .imageRoot.tag | toString -}}
+{{- if .global }}
+    {{- if .global.imageRegistry }}
+     {{- $registryName = .global.imageRegistry -}}
+    {{- end -}}
+{{- end -}}
+{{- if $registryName }}
+{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- else -}}
+{{- printf "%s:%s" $repositoryName $tag -}}
+{{- end -}}
 {{- end -}}
 
 {{/*
@@ -120,40 +140,41 @@ Usage:
 {{- end -}}
 
 
-{{/*
-Return the proper image name
-{{ include "graphscope-store.images.image" . }}
-*/}}
-{{- define "graphscope-store.images.image" -}}
-{{- $tag := .Chart.AppVersion | toString -}}
-{{- with .Values.image -}}
-{{- if .tag -}}
-{{- $tag = .tag | toString -}}
-{{- end -}}
-{{- if .registry -}}
-{{- printf "%s/%s:%s" .registry .repository $tag -}}
-{{- else -}}
-{{- printf "%s:%s" .repository $tag -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
 
 
 {{/*
-Return  the proper Storage Class
-{{ include "graphscope-store.storage.class" .Values.path.to.the.persistence }}
+Return the proper Storage Class
 */}}
-{{- define "graphscope-store.storage.class" -}}
-
-{{- $storageClass := .storageClass -}}
-{{- if $storageClass -}}
-  {{- if (eq "-" $storageClass) -}}
-      {{- printf "storageClassName: \"\"" -}}
-  {{- else }}
-      {{- printf "storageClassName: %s" $storageClass -}}
-  {{- end -}}
+{{- define "graphscope-store.storageClass" -}}
+{{/*
+Helm 2.11 supports the assignment of a value to a variable defined in a different scope,
+but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic.
+*/}}
+{{- if .Values.global -}}
+    {{- if .Values.global.storageClass -}}
+        {{- if (eq "-" .Values.global.storageClass) -}}
+            {{- printf "storageClassName: \"\"" -}}
+        {{- else }}
+            {{- printf "storageClassName: %s" .Values.global.storageClass -}}
+        {{- end -}}
+    {{- else -}}
+        {{- if .Values.persistence.storageClass -}}
+              {{- if (eq "-" .Values.persistence.storageClass) -}}
+                  {{- printf "storageClassName: \"\"" -}}
+              {{- else }}
+                  {{- printf "storageClassName: %s" .Values.persistence.storageClass -}}
+              {{- end -}}
+        {{- end -}}
+    {{- end -}}
+{{- else -}}
+    {{- if .Values.persistence.storageClass -}}
+        {{- if (eq "-" .Values.persistence.storageClass) -}}
+            {{- printf "storageClassName: \"\"" -}}
+        {{- else }}
+            {{- printf "storageClassName: %s" .Values.persistence.storageClass -}}
+        {{- end -}}
+    {{- end -}}
 {{- end -}}
-
 {{- end -}}
 
 {{/*

diff --git a/charts/graphscope-store/templates/configmap.yaml b/charts/graphscope-store/templates/configmap.yaml
@@ -72,9 +72,13 @@ data:
     gaia.rpc.port=60000
     gaia.engine.port=60001
 
+    ## Auth config
+    auth.username={{ .Values.auth.username }}
+    auth.password={{ .Values.auth.password }}
+
     ## Extra Config
     {{- if .Values.extraConfig }}
-    {{- $config_list := regexSplit ":" .Values.extraConfig -1 }}
+    {{- $config_list := regexSplit ";" .Values.extraConfig -1 }}
     {{- range $config_list }}
     {{ . }}
     {{- end }}

diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml
@@ -52,6 +52,9 @@ spec:
       {{- if .Values.coordinator.affinity }}
       affinity: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.affinity "context" $) | nindent 8 }}
       {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
       initContainers:
         {{- if .Values.frontend.initContainers }}
         {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.frontend.initContainers "context" $) | nindent 8 }}
@@ -164,7 +167,7 @@ spec:
         resources:
           requests:
             storage: {{ .Values.coordinator.persistence.size | quote }}
-        {{ include "graphscope-store.storage.class" .Values.coordinator.persistence }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
         {{- if .Values.coordinator.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}

diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml
@@ -52,6 +52,9 @@ spec:
       {{- if .Values.frontend.affinity }}
       affinity: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.frontend.affinity "context" $) | nindent 8 }}
       {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
       initContainers:
         {{- if .Values.frontend.initContainers }}
         {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.frontend.initContainers "context" $) | nindent 8 }}

diff --git a/charts/graphscope-store/templates/ingestor/statefulset.yaml b/charts/graphscope-store/templates/ingestor/statefulset.yaml
@@ -52,6 +52,9 @@ spec:
       {{- if .Values.ingestor.affinity }}
       affinity: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.ingestor.affinity "context" $) | nindent 8 }}
       {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
       initContainers:
         {{- if .Values.ingestor.initContainers }}
         {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.ingestor.initContainers "context" $) | nindent 8 }}

diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml
@@ -52,6 +52,9 @@ spec:
       {{- if .Values.store.affinity }}
       affinity: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.affinity "context" $) | nindent 8 }}
       {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
       initContainers:
         {{- if .Values.store.initContainers }}
         {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.initContainers "context" $) | nindent 8 }}
@@ -173,7 +176,7 @@ spec:
         resources:
           requests:
             storage: {{ .Values.store.persistence.size | quote }}
-        {{ include "graphscope-store.storage.class" .Values.store.persistence }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
         {{- if .Values.store.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}

diff --git a/charts/graphscope-store/templates/test/test-rpc.yaml b/charts/graphscope-store/templates/test/test-rpc.yaml
@@ -5,6 +5,9 @@ metadata:
   annotations:
     "helm.sh/hook": test
 spec:
+  {{- if .Values.tolerations }}
+  tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 4 }}
+  {{- end }}
   containers:
     - name: python
       image: python:3.9.9-alpine3.14
@@ -28,6 +31,9 @@ metadata:
   annotations:
     "helm.sh/hook": test
 spec:
+  {{- if .Values.tolerations }}
+  tolerations: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 4 }}
+  {{- end }}
   containers:
     - name: python
       image: python:3.9.9-alpine3.14

diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml
@@ -81,9 +81,14 @@ executorQueryStoreThreadCount: 2
 log4rsConfig: "LOG4RS_CONFIG"
 
 
-
+## Key-value pair seperated by ;
+## For example extraConfig=k1=v1;k2=v2
 extraConfig: ""
 
+auth:
+  username: ""
+  password: ""
+
 ## Role Based Access
 ## ref: https://kubernetes.io/docs/admin/authorization/rbac/
 ##
@@ -101,6 +106,16 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+persistence:
+    ## Persistent Volume Storage Class
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    storageClass: ""
+
 ## Name of existing ConfigMap with GraphScope configuration.
 ## NOTE: When it's set the 'configuration' parameter is ignored
 ##
@@ -139,16 +154,16 @@ store:
   ##
   affinity: {}
 
-  ## Node labels for GraphScope store pods assignment
-  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
-  ##
-  nodeSelector: {}
-
   ## Tolerations for GraphScope store pods assignment
   ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
   ##
   tolerations: []
 
+  ## Node labels for GraphScope store pods assignment
+  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
+  ##
+  nodeSelector: {}
+
   ## GraphScope store Pod security context
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   ##
@@ -207,14 +222,6 @@ store:
     ## NOTE: When it's set the rest of persistence parameters are ignored
     ##
     # existingClaim:
-    ## Persistent Volume Storage Class
-    ## If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
-    ##
-    storageClass: ""
     ## Persistent Volume Claim annotations
     ##
     annotations: {}
@@ -301,16 +308,16 @@ frontend:
   ##
   affinity: {}
 
-  ## Node labels for GraphScope store pods assignment
-  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
-  ##
-  nodeSelector: {}
-
   ## Tolerations for GraphScope store pods assignment
   ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
   ##
   tolerations: []
 
+  ## Node labels for GraphScope store pods assignment
+  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
+  ##
+  nodeSelector: {}
+
   ## GraphScope store container's resource requests and limits
   ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
   ##
@@ -450,16 +457,16 @@ ingestor:
   ##
   affinity: {}
 
-  ## Node labels for GraphScope store pods assignment
-  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
-  ##
-  nodeSelector: {}
-
   ## Tolerations for GraphScope store pods assignment
   ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
   ##
   tolerations: []
 
+  ## Node labels for GraphScope store pods assignment
+  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
+  ##
+  nodeSelector: {}
+
   ## GraphScope store container's resource requests and limits
   ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
   ##
@@ -562,16 +569,16 @@ coordinator:
   ##
   affinity: {}
 
-  ## Node labels for GraphScope store pods assignment
-  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
-  ##
-  nodeSelector: {}
-
   ## Tolerations for GraphScope store pods assignment
   ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
   ##
   tolerations: []
 
+  ## Node labels for GraphScope store pods assignment
+  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
+  ##
+  nodeSelector: {}
+
   ## GraphScope store container's resource requests and limits
   ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
   ##
@@ -618,14 +625,6 @@ coordinator:
     ## NOTE: When it's set the rest of persistence parameters are ignored
     ##
     # existingClaim:
-    ## Persistent Volume Storage Class
-    ## If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
-    ##
-    storageClass: ""
     ## Persistent Volume Claim annotations
     ##
     annotations: {}
@@ -699,4 +698,5 @@ externalKafka:
   servers: []
 
 global:
+  imageRegistry: ""
   storageClass: ""