[GIE Standalone] Add helm chart to deploy GIE on vineyard storage

.github/workflows/k8s-ci.yml

@@ -88,7 +88,7 @@ jobs:
             - '.github/workflows/ci.yml'
           gie-k8s-failover-test:
             - 'interactive_engine/**'
-            - 'charts/ir-standalone/**'
+            - 'charts/gie-standalone/**'
 
     - name: Cpp Format and Lint Check
       run: |
@@ -647,13 +647,16 @@ jobs:
         run: |
           cd ${GITHUB_WORKSPACE}/charts
           # create local persistent volume which contains graph data for test
-          kubectl apply -f ./ir-standalone/tools/pvc.yaml
+          kubectl apply -f ./gie-standalone/tools/pvc.yaml
           # create gie instance (compiler & executor & exp storage)
-          helm install test ./ir-standalone \
-              --set image.repository=graphscope/interactive-experimental \
-              --set image.tag=${SHORT_SHA} --set storageType=Experimental \
+          helm install test ./gie-standalone \
+              --set frontend.image.repository=graphscope/interactive-experimental \
+              --set frontend.image.tag=${SHORT_SHA} \
+              --set executor.image.repository=graphscope/interactive-experimental \
+              --set executor.image.tag=${SHORT_SHA} \
+              --set storageType=Experimental \
               --set schemaConfig=expr_modern_schema.json \
-              --set store.replicaCount=2 \
+              --set executor.replicaCount=2 \
               --set frontend.service.type=NodePort
           # run failover test
-          cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-graphscope-store 2 1
+          cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-gie-standalone 2 1

.github/workflows/release.yml

@@ -103,6 +103,8 @@ jobs:
         helm package ./graphscope
         # package graphscope-store
         helm package --dependency-update ./graphscope-store
+        # package gie-standalone
+        helm package --dependency-update ./gie-standalone
         # generate index.yaml
         helm repo index . --url ${{ env.HELM_CHARTS_URL }}
 
@@ -121,4 +123,5 @@ jobs:
         assets: |
           charts/graphscope-${{ steps.tag.outputs.TAG }}.tgz:/charts/graphscope-${{ steps.tag.outputs.TAG }}.tgz
           charts/graphscope-store-${{ steps.tag.outputs.TAG }}.tgz:/charts/graphscope-store-${{ steps.tag.outputs.TAG }}.tgz
+          charts/gie-standalone-${{ steps.tag.outputs.TAG }}.tgz:/charts/gie-standalone-${{ steps.tag.outputs.TAG }}.tgz
           charts/index.yaml:/charts/index.yaml

charts/ir-standalone/Chart.yaml → charts/gie-standalone/Chart.yaml

@@ -1,16 +1,18 @@
 annotations:
   category: Database
+  licenses: Apache-2.0
 apiVersion: v2
-appVersion: latest
-description: Chart to create a GraphScope Store cluster (Ir on Vineyard)
+appVersion: 3.4.0
+description: Chart to Deploy GIE on Vineyard Storage
 home: https://graphscope.io
+icon: https://graphscope.io/assets/images/graphscope-logo.svg
 keywords:
   - GraphScope
   - database
   - cluster
-name: graphscope-store
+name: gie-standalone
 sources:
   - https://github.com/alibaba/GraphScope/tree/main/interactive_engine/
-version: v2
+version: 0.20.0
 
 dependencies:

charts/gie-standalone/README.md

@@ -0,0 +1,173 @@
+# GIE Standalone Deployment
+## Prerequisite
+### etcd
+```
+# TODO: the link need to be updated after merging to main
+kubectl apply -f https://raw.githubusercontent.com/shirly121/GraphScope/add_gie_deploy/charts/gie-standalone/tools/etcd.yaml
+```
+### prepare graph data
+- vineyard
+```
+# there are some sample data for tests under the `gie-standalone` directory, just copy them to the target directory
+
+data
+└── modern_graph
+    ├── created.csv
+    ├── knows.csv
+    ├── person.csv
+    └── software.csv
+
+helm pull graphscope/gie-standalone --untar
+cp -r gie-standalone/data/ /tmp/data/
+```
+- experimental
+```
+# experimental storage will create modern graph for tests by default,
+# prepare your own data under the directories of `graph_schema` and `graph_data_bin` if needed.
+cp -r graph_schema /tmp/data/
+cp -r graph_data_bin /tmp/data
+```
+### prepare k8s volume
+- download `pvc.yaml`
+```
+# TODO: the link need to be updated after merging to main
+curl -O -S https://raw.githubusercontent.com/shirly121/GraphScope/add_gie_deploy/charts/gie-standalone/tools/pvc.yaml
+```
+- config `pvc.yaml`
+```
+hostPath:
+  path: /tmp/data # keep consistent with the directory where the graph data is located
+```
+- create pvc and pv
+```
+kubectl apply -f pvc.yaml
+```
+## Getting Started
+### start gie deployment
+- from remote
+```
+helm repo update
+helm install <your-release-name> graphscope/gie-standalone
+```
+- from local (for customized config)
+```
+# download helm package
+helm pull graphscope/gie-standalone --untar
+helm install <your-release-name> gie-standalone
+```
+### stop gie deployment
+```
+helm delete <your-release-name>
+```
+### get service endpoint
+```
+# execute in advance if in minikube environment
+minikube tunnel 
+# gremlin endpoint
+kubectl describe svc <your-release-name>-gie-standalone-frontend | grep "Endpoints:" | awk -F' ' '{print $2}'
+```
+## Customized Config
+### download helm package
+```
+helm pull graphscope/gie-standalone --untar 
+
+gie-standalone/
+├── Chart.yaml
+├── config
+│   └── v6d_modern_loader.json
+├── data
+│   └── modern_graph
+│       ├── created.csv
+│       ├── knows.csv
+│       ├── person.csv
+│       └── software.csv
+├── README.md
+├── templates
+│   ├── configmap.yaml
+│   ├── frontend
+│   │   ├── statefulset.yaml
+│   │   └── svc.yaml
+│   ├── _helpers.tpl
+│   ├── serviceaccount.yaml
+│   └── store
+│       ├── statefulset.yaml
+│       └── svc-headless.yaml
+├── tools
+│   ├── etcd.yaml
+│   └── pvc.yaml
+└── values.yaml
+```
+### config `gie-standalone/values.yaml`
+#### vineyard
+- docker artifacts
+```
+# docker artifacts for vineyard store
+executor:
+  image:
+    registry: registry.cn-hongkong.aliyuncs.com
+    repository: graphscope/interactive-executor
+    tag: "0.20.0"
+
+# docker artifacts for frontend
+frontend:
+  image:
+    registry: registry.cn-hongkong.aliyuncs.com
+    repository: graphscope/interactive-frontend
+    tag: "0.20.0"
+```
+- load your own graph data
+```
+# add your graph loader config under `gie-standalone/config`
+
+config
+└── v6d_modern_loader.json
+
+# config the file name in `gie-standalone/values.yaml`
+htapLoaderConfig: "v6d_modern_loader.json" 
+```
+#### common
+```
+frontend:
+  replicaCount: 1 # frontend num
+    service:
+      gremlinPort: 8182 # gremlin service port
+
+executor:
+  replicaCount: 1 # executor num
+
+# job config
+pegasusWorkerNum: 2
+pegasusTimeout: 240000
+pegasusBatchSize: 1024
+pegasusOutputCapacity: 16
+
+# k8s volume to store graph data
+existingClaim: "test-graphscope-store-pvc"
+
+# data path where the inner pod read graph data from
+storeDataPath: "/tmp/data"
+# hdfs path is supported in vineyard
+# storeDataPath: "hdfs://{ip}:{port}"
+```
+#### experimental
+```
+# docker artifacts for experimental store
+executor:
+  image:
+    registry: registry.cn-hongkong.aliyuncs.com
+    repository: graphscope/gie-exp-runtime
+    tag: ""
+
+# docker artifacts for frontend
+frontend:
+  image:
+    registry: registry.cn-hongkong.aliyuncs.com
+    repository: graphscope/gie-exp-runtime
+    tag: ""
+
+# storage type
+storageType: Experimental
+
+# schema needed by compiler, config in `gie-standalone/templates/configmap.yaml`
+schemaConfig: "expr_modern_schema.json"
+```

charts/gie-standalone/config/v6d_modern_loader.json

@@ -0,0 +1,33 @@
+{
+    "vertices": [
+        {
+            "data_path": "$STORE_DATA_PATH/modern_graph/person.csv",
+            "label": "person",
+            "options": "header_row=true&delimiter=|"
+        },
+        {
+            "data_path": "$STORE_DATA_PATH/modern_graph/software.csv",
+            "label": "software",
+            "options": "header_row=true&delimiter=|"
+        }
+    ],
+    "edges": [
+        {
+            "data_path": "$STORE_DATA_PATH/modern_graph/knows.csv",
+            "label": "knows",
+            "src_label": "person",
+            "dst_label": "person",
+            "options": "header_row=true&delimiter=|"
+        },
+        {
+            "data_path": "$STORE_DATA_PATH/modern_graph/created.csv",
+            "label": "created",
+            "src_label": "person",
+            "dst_label": "software",
+            "options": "header_row=true&delimiter=|"
+        }
+    ],
+    "directed": 1,
+    "retain_oid": 1,
+    "generate_eid": 1
+}

charts/gie-standalone/data/modern_graph/created.csv

@@ -0,0 +1,5 @@
+src_id|dst_id|edge_id|weight
+1|3|9|0.4
+4|5|10|1.0
+4|3|11|0.4
+6|3|12|0.2

charts/gie-standalone/data/modern_graph/knows.csv

@@ -0,0 +1,3 @@
+src_id|dst_id|edge_id|weight
+1|2|7|0.5
+1|4|8|1.0

charts/gie-standalone/data/modern_graph/person.csv

@@ -0,0 +1,5 @@
+id|name|age
+2|vadas|27
+6|peter|35
+4|josh|32
+1|marko|29

charts/gie-standalone/data/modern_graph/software.csv

@@ -0,0 +1,3 @@
+id|name|lang
+3|lop|java
+5|ripple|java

charts/ir-standalone/templates/_helpers.tpl → charts/gie-standalone/templates/_helpers.tpl

@@ -71,11 +71,39 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
-Return the proper graphscope-store image name
+Return the proper graphscope-store frontend image name
 */}}
-{{- define "graphscope-store.image" -}}
-{{ include "graphscope-store.images.image" . }}
+{{- define "graphscope-store.frontend.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.frontend.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper graphscope-store store image name
+*/}}
+{{- define "graphscope-store.store.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.executor.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
 {{- end -}}
+{{- end -}}
+
 
 {{/*
 Create the name of the service account to use
@@ -124,25 +152,6 @@ Usage:
 {{- end -}}
 
 
-{{/*
-Return the proper image name
-{{ include "graphscope-store.images.image" . }}
-*/}}
-{{- define "graphscope-store.images.image" -}}
-{{- $tag := .Chart.AppVersion | toString -}}
-{{- with .Values.image -}}
-{{- if .tag -}}
-{{- $tag = .tag | toString -}}
-{{- end -}}
-{{- if .registry -}}
-{{- printf "%s/%s:%s" .registry .repository $tag -}}
-{{- else -}}
-{{- printf "%s:%s" .repository $tag -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-
 {{/*
 Return  the proper Storage Class
 {{ include "graphscope-store.storage.class" .Values.path.to.the.persistence }}