[GIE Dev Test] Add docs for interactive dev test (#2733)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
1. add docs for interactive dev test.
2. refine gs cmd to make or test gie in local dev mode.

<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: longbinlai <longbin.lai@gmail.com>
Co-authored-by: Longbin Lai <longbin.lailb@alibaba-inc.com>

.github/workflows/gaia.yml

@@ -85,7 +85,7 @@ jobs:
     - name: Ir Integration Pattern Test on Experimental Store
       run: |
         cd ${GITHUB_WORKSPACE}
-        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git
+        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git /tmp/gstest
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_pattern_ci.sh
 
     - name: Ir Integration Test on Csr Store

docs/index.rst

@@ -64,6 +64,7 @@ and the vineyard store that offers efficient in-memory data transfers.
    interactive_engine/guide_and_examples
    interactive_engine/design_of_gie
    interactive_engine/supported_gremlin_steps
+   interactive_engine/dev_and_test.md
    interactive_engine/faq.md
 
 .. toctree::

docs/interactive_engine/dev_and_test.md

@@ -0,0 +1,45 @@
+# Dev and Test
+
+This document describes how to build and test GraphScope Interactive Engine from source code.
+
+## Dev Environment
+
+Here we would use a prebuilt docker image with necessary dependencies installed.
+
+```bash
+docker run --name dev -it --shm-size=4096m registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:latest
+```
+
+Please refer to [Dev Environment](../development/dev_guide.md#dev-environment) to find more options to get a dev environment.
+
+## Build Interactive Engine
+
+With `gs` command-line utility, you can build interactive engine of GraphScope with a single command.
+
+```bash
+# Clone a repo if needed
+# git clone https://github.com/alibaba/graphscope
+# cd graphscope
+./gs make interactive --experimental
+```
+
+You may want to grab a cup of coffee cause this compiling will take a while, which
+includes compiling the java code of GIE compiler, and the rust code of GIE engine.
+You may found the built artifacts in `interactive_engine/assembly/target/graphscope.tar.gz`.
+
+## How to Test
+
+You could easily test with the new artifacts with a single command:
+
+Here we set the working directory to local repo.
+```bash
+export GRAPHSCOPE_HOME=`pwd`
+# Here the `pwd` is the root path of GraphScope repository
+```
+See more about `GRAPHSCOPE_HOME` in [run tests](../development/how_to_test.md#run-tests)
+
+```bash
+./gs test interactive
+```
+
+It would download the test dataset to the `/tmp/gstest` (if not exists) and run multiple algorithms against various graphs, and compare the result with the ground truth.

gs

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# This script was generated by bashly 1.0.2 (https://bashly.dannyb.co)
+# This script was generated by bashly 1.0.4 (https://bashly.dannyb.co)
 # Modifying it manually is not recommended
 
 # :wrapper.bash3_bouncer
@@ -103,6 +103,11 @@ gs_make_usage() {
     printf "    Default: /opt/graphscope\n"
     echo
 
+    # :flag.usage
+    printf "  %s\n" "--experimental"
+    printf "    make install experimental components\n"
+    echo
+
     # :command.usage_fixed_flags
     printf "  %s\n" "--help, -h"
     printf "    Show this help\n"
@@ -1027,7 +1032,7 @@ install_vineyard() {
   make -j"${jobs}"
   make install
   strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard*
-  pip3 install "vineyard==${v6d_version}"
+  pip3 install --no-cache -i https://pypi.org/simple "vineyard==${v6d_version}"
   cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/
   popd || exit
   popd || exit
@@ -1135,6 +1140,9 @@ gs_make_command() {
   log "Making component ${component}"
 
   install_prefix=${args[--install-prefix]}
+  experimental_flag=${args[--experimental]}
+
+  GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")"
 
   export INSTALL_PREFIX=${install_prefix}
 
@@ -1151,7 +1159,12 @@ gs_make_command() {
   }
 
   make_interactive() {
-      make interactive
+      if [[ -n ${experimental_flag} ]]; then
+          cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make build QUIET_OPT=""
+
+      else
+          make interactive
+      fi
   }
 
   make_learning() {
@@ -1339,7 +1352,7 @@ gs_test_command() {
     get_test_data
     if [[ -n ${on_local} ]]; then
       # IR unit test
-      cd interactive_engine/compiler && make test
+      cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make test
       # CommonType Unit Test
       cd "${GS_SOURCE_DIR}"/interactive_engine/executor/common/dyn_type && cargo test
       # Store Unit test
@@ -2004,6 +2017,14 @@ gs_make_parse_requirements() {
         fi
         ;;
 
+      # :flag.case
+      --experimental)
+
+        # :flag.case_no_arg
+        args['--experimental']=1
+        shift
+        ;;
+
       -?*)
         printf "invalid option: %s\n" "$key" >&2
         exit 1
@@ -2031,7 +2052,7 @@ gs_make_parse_requirements() {
   [[ -n ${args['--install-prefix']:-} ]] || args['--install-prefix']="/opt/graphscope"
 
   # :command.whitelist_filter
-  if [[ ! ${args['component']} =~ ^(all|install|analytical|analytical-java|interactive|learning|analytical-install|analytical-java-install|interactive-install|learning-install|client|coordinator|clean)$ ]]; then
+  if [[ -n ${args['component']} ]] && [[ ! ${args['component']} =~ ^(all|install|analytical|analytical-java|interactive|learning|analytical-install|analytical-java-install|interactive-install|learning-install|client|coordinator|clean)$ ]]; then
     printf "%s\n" "component must be one of: all, install, analytical, analytical-java, interactive, learning, analytical-install, analytical-java-install, interactive-install, learning-install, client, coordinator, clean" >&2
     exit 1
   fi
@@ -2149,7 +2170,7 @@ gs_make_image_parse_requirements() {
   [[ -n ${args['--tag']:-} ]] || args['--tag']="latest"
 
   # :command.whitelist_filter
-  if [[ ! ${args['image']} =~ ^(all|graphscope-dev|coordinator|analytical|analytical-java|interactive|interactive-frontend|interactive-executor|learning|vineyard-dev|vineyard-runtime)$ ]]; then
+  if [[ -n ${args['image']} ]] && [[ ! ${args['image']} =~ ^(all|graphscope-dev|coordinator|analytical|analytical-java|interactive|interactive-frontend|interactive-executor|learning|vineyard-dev|vineyard-runtime)$ ]]; then
     printf "%s\n" "image must be one of: all, graphscope-dev, coordinator, analytical, analytical-java, interactive, interactive-frontend, interactive-executor, learning, vineyard-dev, vineyard-runtime" >&2
     exit 1
   fi
@@ -2312,7 +2333,7 @@ gs_test_parse_requirements() {
   [[ -n ${args['--testdata']:-} ]] || args['--testdata']="/tmp/gstest"
 
   # :command.whitelist_filter
-  if [[ ! ${args['type']} =~ ^(analytical|analytical-java|interactive|learning|local-e2e|k8s-e2e|groot)$ ]]; then
+  if [[ -n ${args['type']} ]] && [[ ! ${args['type']} =~ ^(analytical|analytical-java|interactive|learning|local-e2e|k8s-e2e|groot)$ ]]; then
     printf "%s\n" "type must be one of: analytical, analytical-java, interactive, learning, local-e2e, k8s-e2e, groot" >&2
     exit 1
   fi
@@ -2462,7 +2483,7 @@ gs_install_deps_parse_requirements() {
   [[ -n ${args['--v6d-version']:-} ]] || args['--v6d-version']="main"
 
   # :command.whitelist_filter
-  if [[ ! ${args['type']} =~ ^(dev|client)$ ]]; then
+  if [[ -n ${args['type']} ]] && [[ ! ${args['type']} =~ ^(dev|client)$ ]]; then
     printf "%s\n" "type must be one of: dev, client" >&2
     exit 1
   fi
@@ -2524,7 +2545,7 @@ gs_format_parse_requirements() {
   fi
 
   # :command.whitelist_filter
-  if [[ ! ${args['lang']} =~ ^(cpp|java|python|rust)$ ]]; then
+  if [[ -n ${args['lang']} ]] && [[ ! ${args['lang']} =~ ^(cpp|java|python|rust)$ ]]; then
     printf "%s\n" "lang must be one of: cpp, java, python, rust" >&2
     exit 1
   fi

interactive_engine/compiler/Makefile

@@ -18,9 +18,11 @@ rpc.target:=start_rpc_server
 
 target.revision:=0.0.1-SNAPSHOT
 
+QUIET_OPT := --quiet
+
 build:
 	cd $(CUR_DIR)/.. && \
-		mvn clean install -DskipTests -Drevision=${target.revision} -Pexperimental --quiet && \
+		mvn clean install -DskipTests -Drevision=${target.revision} -Pexperimental ${QUIET_OPT} && \
 		cd $(CUR_DIR)/../executor/ir/integrated && cargo build --release --bin $(rpc.target)
 
 clean:

interactive_engine/compiler/ir_csr_ci.sh

@@ -2,7 +2,7 @@
 base_dir=$(cd $(dirname $0); pwd)
 # start engine service and load modern graph
 cd ${base_dir}/../executor/ir/target/release &&
-RUST_LOG=info CSR_PATH=${base_dir}/../../gstest/modern_graph_csr_bin PARTITION_ID=0 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config &
+RUST_LOG=info CSR_PATH=/tmp/gstest/modern_graph_csr_bin PARTITION_ID=0 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config &
 sleep 5s
 # start compiler service
 cd ${base_dir} && make run graph.store=csr &

interactive_engine/compiler/ir_csr_pattern_ci.sh

@@ -6,9 +6,9 @@ ps -ef | grep "start_rpc_server" |  awk '{print $2}' | xargs kill -9
 sleep 3
 # start engine service and load ldbc graph
 cd ${base_dir}/../executor/ir/target/release &&
-RUST_LOG=info CSR_PATH=${base_dir}/../../gstest/ldbc_graph_csr_bin PARTITION_ID=0 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config/distributed/server_0 &
+RUST_LOG=info CSR_PATH=/tmp/gstest/ldbc_graph_csr_bin PARTITION_ID=0 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config/distributed/server_0 &
 cd ${base_dir}/../executor/ir/target/release &&
-RUST_LOG=info CSR_PATH=${base_dir}/../../gstest/ldbc_graph_csr_bin PARTITION_ID=1 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config/distributed/server_1 &
+RUST_LOG=info CSR_PATH=/tmp/gstest/ldbc_graph_csr_bin PARTITION_ID=1 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config/distributed/server_1 &
 sleep 10
 # start compiler service
 cd ${base_dir} && make run graph.schema:=../executor/ir/core/resource/ldbc_schema_csr.json pegasus.hosts:=127.0.0.1:1234,127.0.0.1:1235 &

interactive_engine/compiler/ir_exprimental_pattern_ci.sh

@@ -6,9 +6,9 @@ ps -ef | grep "start_rpc_server" |  awk '{print $2}' | xargs kill -9
 sleep 3
 # start engine service and load ldbc graph
 cd ${base_dir}/../executor/ir/target/release &&
-RUST_LOG=info DATA_PATH=${base_dir}/../../gstest/ldbc_graph_exp_bin PARTITION_ID=0 ./start_rpc_server --config ${base_dir}/../executor/ir/integrated/config/distributed/server_0 &
+RUST_LOG=info DATA_PATH=/tmp/gstest/ldbc_graph_exp_bin PARTITION_ID=0 ./start_rpc_server --config ${base_dir}/../executor/ir/integrated/config/distributed/server_0 &
 cd ${base_dir}/../executor/ir/target/release &&
-RUST_LOG=info DATA_PATH=${base_dir}/../../gstest/ldbc_graph_exp_bin PARTITION_ID=1 ./start_rpc_server --config ${base_dir}/../executor/ir/integrated/config/distributed/server_1 &
+RUST_LOG=info DATA_PATH=/tmp/gstest/ldbc_graph_exp_bin PARTITION_ID=1 ./start_rpc_server --config ${base_dir}/../executor/ir/integrated/config/distributed/server_1 &
 sleep 10
 # start compiler service
 cd ${base_dir} && make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json pegasus.hosts:=127.0.0.1:1234,127.0.0.1:1235 &