Add persistent storage (#361)

Signed-off-by: 海虾 <tianli.tl@alibaba-inc.com> Co-authored-by: BingqingLyu <lv_bingqing@163.com> Co-authored-by: Siyuan Zhang <siyuan0322@gmail.com> Co-authored-by: Tao He <linzhu.ht@alibaba-inc.com>
alibaba · May 31, 2021 · 28d4b3b · 28d4b3b
1 parent 0226e00
commit 28d4b3b
Show file tree

Hide file tree

Showing 1,068 changed files with 147,533 additions and 461 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -144,7 +144,6 @@ jobs:
       run: |
         yum install -y http://opensource.wandisco.com/centos/7/git/x86_64/wandisco-git-release-7-2.noarch.rpm
         yum install -y git
-
         wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz
         tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz
         curl -sf -L https://static.rust-lang.org/rustup.sh | sh -s -- -y --profile minimal --default-toolchain 1.48.0

diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml
@@ -54,6 +54,11 @@ jobs:
         sudo apt install -y libarrow-dev=1.0.1-1 libarrow-python-dev=1.0.1-1
 
         # install zookeeper
+        #wget https://archive.apache.org/dist/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
+        #tar xf apache-zookeeper-3.6.2-bin.tar.gz -C /tmp/
+        #cp /tmp/apache-zookeeper-3.6.2-bin/conf/zoo_sample.cfg /tmp/apache-zookeeper-3.6.2-bin/conf/zoo.cfg
+        #sudo ln -s /tmp/apache-zookeeper-3.6.2-bin /usr/local/zookeeper
+
         wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz
         tar xf zookeeper-3.4.14.tar.gz -C /tmp/
         cp /tmp/zookeeper-3.4.14/conf/zoo_sample.cfg /tmp/zookeeper-3.4.14/conf/zoo.cfg
@@ -121,6 +126,9 @@ jobs:
         export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} 
         cp ${GITHUB_WORKSPACE}/python/tests/test_run_locally.py /tmp/test_run_locally.py
+        rm -rf ~/.ccache
+        rm -rf ~/.m2
+        rm -rf ~/.cargo
         python3 -m pytest -s -v /tmp/test_run_locally.py
 
     - name: Setup tmate session debug

diff --git a/.gitignore b/.gitignore
@@ -52,3 +52,12 @@ default.etcd/
 
 # outer build directory
 build/
+
+interactive_engine/src/common/rust/common/src/proto/*
+!interactive_engine/src/common/rust/common/src/proto/mod.rs
+**/generated/
+
+interactive_engine/src/executor/store/src/db/proto/*
+!interactive_engine/src/executor/store/src/db/proto/mod.rs
+interactive_engine/src/data_load_tools/dependency-reduced-pom.xml
+
diff --git a/charts/graphscope-store-service/.helmignore b/charts/graphscope-store-service/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/graphscope-store-service/Chart.lock b/charts/graphscope-store-service/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: kafka
+  repository: https://charts.bitnami.com/bitnami
+  version: 12.17.6
+digest: sha256:b2a5473c7a02a74863d85608915d440c3b6e0e2225e2a8c22323cce0e0a5e52d
+generated: "2021-05-25T13:10:43.980721587+08:00"
diff --git a/charts/graphscope-store-service/Chart.yaml b/charts/graphscope-store-service/Chart.yaml
@@ -0,0 +1,30 @@
+apiVersion: v2
+name: graphscope-store-service
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
+
+dependencies:
+  - condition: kafka.enabled
+    name: kafka
+    version: 12.17.6
+    repository: https://charts.bitnami.com/bitnami
diff --git a/charts/graphscope-store-service/README.md b/charts/graphscope-store-service/README.md
@@ -0,0 +1,11 @@
+## GraphScope Store Service
+---
+
+### Introduction
+
+### TL;DR
+
+```shell
+helm dependency update .
+helm install my-release .
+```
diff --git a/charts/graphscope-store-service/templates/NOTES.txt b/charts/graphscope-store-service/templates/NOTES.txt
@@ -0,0 +1,9 @@
+1. Get the application URL by running these commands:
+
+{{- if contains "NodePort" .Values.frontend.service.type }}
+  export GRPC_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "graphscope-store-service.frontend.fullname" . }})
+  export GREMLIN_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[1].nodePort}" services {{ include "graphscope-store-service.frontend.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo "GRPC endpoint is: http://$NODE_IP:$GRPC_PORT"
+  echo "GREMLIN endpoint is: http://$NODE_IP:$GREMLIN_PORT"
+{{- end }}
diff --git a/charts/graphscope-store-service/templates/_helpers.tpl b/charts/graphscope-store-service/templates/_helpers.tpl
@@ -0,0 +1,184 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "graphscope-store-service.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "graphscope-store-service.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "graphscope-store-service.coordinator.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-store-service.fullname" .) "coordinator" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "graphscope-store-service.frontend.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-store-service.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "graphscope-store-service.ingestor.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-store-service.fullname" .) "ingestor" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "graphscope-store-service.store.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-store-service.fullname" .) "store" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "graphscope-store-service.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "graphscope-store-service.labels" -}}
+helm.sh/chart: {{ include "graphscope-store-service.chart" . }}
+{{ include "graphscope-store-service.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "graphscope-store-service.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "graphscope-store-service.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Return the proper graphscope-store-service image name
+*/}}
+{{- define "graphscope-store-service.image" -}}
+{{ include "graphscope-store-service.images.image" .Values.image }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "graphscope-store-service.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "graphscope-store-service.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Return the configmap with the graphscope configuration
+*/}}
+{{- define "graphscope-store-service.configmapName" -}}
+{{- if .Values.existingConfigmap -}}
+    {{- printf "%s" (tpl .Values.existingConfigmap $) -}}
+{{- else -}}
+    {{- printf "%s" (include "graphscope-store-service.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a configmap object should be created for graphscope-service
+*/}}
+{{- define "graphscope-store-service.createConfigmap" -}}
+{{- if not .Values.existingConfigmap }}
+    {{- true -}}
+{{- else -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper image name (for the init container volume-permissions image)
+*/}}
+{{- define "graphscope-store-service.volumePermissions.image" -}}
+{{ include "graphscope-store-service.images.image" .Values.volumePermissions.image }}
+{{- end -}}
+
+{{/*
+Renders a value that contains template.
+Usage:
+{{ include "graphscope-store-service.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }}
+*/}}
+{{- define "graphscope-store-service.tplvalues.render" -}}
+    {{- if typeIs "string" .value }}
+        {{- tpl .value .context }}
+    {{- else }}
+        {{- tpl (.value | toYaml) .context }}
+    {{- end }}
+{{- end -}}
+
+
+{{/*
+Return the proper image name
+{{ include "graphscope-store-service.images.image" .Values.path.to.the.image }}
+*/}}
+{{- define "graphscope-store-service.images.image" -}}
+{{- $tag := .tag | toString -}}
+{{- if .registry }}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Return  the proper Storage Class
+{{ include "graphscope-store-service.storage.class" .Values.path.to.the.persistence }}
+*/}}
+{{- define "graphscope-store-service.storage.class" -}}
+
+{{- $storageClass := .storageClass -}}
+{{- if $storageClass -}}
+  {{- if (eq "-" $storageClass) -}}
+      {{- printf "storageClassName: \"\"" -}}
+  {{- else }}
+      {{- printf "storageClassName: %s" $storageClass -}}
+  {{- end -}}
+{{- end -}}
+
+{{- end -}}
+
+{{/*
+Create a default fully qualified kafka name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "graphscope-store-service.kafka.fullname" -}}
+{{- printf "%s-%s" .Release.Name "kafka" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Get full broker list.
+*/}}
+{{- define "graphscope-store-service.kafka.brokerlist" -}}
+
+{{- $replicaCount := int .Values.kafka.replicaCount -}}
+{{- $releaseNamespace := .Release.Namespace -}}
+{{- $clusterDomain := .Values.clusterDomain -}}
+{{- $fullname := include "graphscope-store-service.kafka.fullname" . -}}
+{{- $servicePort := int .Values.kafka.service.port -}}
+
+{{- $brokerList := list }}
+{{- range $e, $i := until $replicaCount }}
+{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }}
+{{- end }}
+{{- join "," $brokerList | printf "%s" -}}
+{{- end -}}
diff --git a/charts/graphscope-store-service/templates/configmap.yaml b/charts/graphscope-store-service/templates/configmap.yaml
@@ -0,0 +1,67 @@
+{{- if (include "graphscope-store-service.createConfigmap" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "graphscope-store-service.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{- include "graphscope-store-service.labels" . | nindent 4 }}
+    app.kubernetes.io/component: configmap
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  my.cnf: |-
+    ## Common Config
+    graph.name={{ .Values.graphName }}
+    role.name=ROLE
+    node.idx=INDEX
+    rpc.port=55555
+    rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }}
+    store.node.count={{ .Values.store.replicaCount }}
+    frontend.node.count={{ .Values.frontend.replicaCount }}
+    ingestor.node.count={{ .Values.ingestor.replicaCount }}
+    coordinator.node.count={{ .Values.coordinator.replicaCount }}
+    ingestor.queue.count={{ .Values.ingestor.replicaCount }}
+    partition.count={{ .Values.store.replicaCount | mul 8 }}
+    engine.type={{ .Values.engineType }}
+    discovery.mode={{ .Values.discoveryMode }}
+
+    ## Ingestor Config
+    ingestor.queue.buffer.size={{ .Values.ingestorQueueBufferSize }}
+    ingestor.sender.buffer.size={{ .Values.ingestorSenderBufferSize }}
+
+    ## Coordinator Config
+    snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }}
+    offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }}
+    file.meta.store.path={{ .Values.fileMetaStorePath }}
+
+    ## Store Config
+    store.data.path={{ .Values.storeDataPath }}
+    store.write.thread.count={{ .Values.storeWriteThreadCount }}
+
+    ## Kafka Config
+    kafka.servers=KAFKA_SERVERS
+    kafka.topic={{ .Values.kafkaTopic }}
+
+    ## Frontend Config
+    graph.store.type={{ .Values.graphStoreType }}
+    gremlin.server.port=12312
+
+    executor.worker.per.process={{ .Values.executorWorkerPerProcess }}
+    executor.query.thread.count={{ .Values.executorQueryThreadCount }}
+    executor.query.manager.thread.count={{ .Values.executorQueryManagerThreadCount }}
+    executor.query.store.thread.count={{ .Values.executorQueryStoreThreadCount }}
+    log4rs.config={{ .Values.log4rsConfig }}
+
+    dns.name.prefix.frontend=FRONTEND
+    dns.name.prefix.ingestor=INGESTOR
+    dns.name.prefix.coordinator=COORDINATOR
+    dns.name.prefix.store=STORE
+
+    executor.graph.port=55556
+    executor.query.port=55557
+    executor.engine.port=55558
+{{- end -}}