feat(interactive): Implement admin http service for hqps engine

.github/workflows/flex.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8
     steps:
     - uses: actions/checkout@v3
 
@@ -44,6 +44,8 @@ jobs:
       env:
         HOME: /home/graphscope/
       run: |
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
         cmake .. && sudo make -j$(nproc)
@@ -74,10 +76,10 @@ jobs:
         GLOG_v=10 ./bin/graph_loader  ${SCHEMA_FILE} ${BULK_LOAD_FILE} /tmp/csr-data-dir/
 
     - name: Test Graph Loading on type_test graph
-    env:
+      env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest/
         FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/type_test/
-    run: |
+      run: |
         # remove modern graph indices
         rm -rf /tmp/csr-data-dir/
 

.github/workflows/hqps-db-ci.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8
     steps:
     - uses: actions/checkout@v3
 
@@ -68,6 +68,8 @@ jobs:
         GIE_HOME: ${{ github.workspace }}/interactive_engine/
         HOME: /home/graphscope/
       run: |
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
         cmake .. && sudo make -j$(nproc)
@@ -90,9 +92,27 @@ jobs:
         git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
         GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
+        BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
         cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
+        cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
         cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
+        cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml
+
+        # load graph
+        cd ${GITHUB_WORKSPACE}/flex/build
+        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
+        GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/
+        export FLEX_DATA_DIR=../interactive/examples/movies
+        GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/indices/
+
+    - name: Test HQPS admin http service
+      env: 
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
+      run:
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
+        bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR}
 
     - name: Sample Query test
       env:
@@ -110,13 +130,12 @@ jobs:
         HOME : /home/graphscope/
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
-        GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine
         cd ${GITHUB_WORKSPACE}/flex/bin
 
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd}
@@ -125,19 +144,19 @@ jobs:
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd}
         done
 
         # test movie graph, 8,9,10 are not supported now
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml
         for i in 1 2 3 4 5 6 7 11 12 13 14 15;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
           echo $cmd
           eval ${cmd}
@@ -150,12 +169,11 @@ jobs:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
-        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to ldbc
-        sed -i 's/default_graph: movies/default_graph: ldbc/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \
-              ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ./engine_config_test.yaml to ldbc
+        sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
+        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
+              ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
     - name: Run End-to-End cypher adhoc movie query test
       env:
@@ -164,11 +182,10 @@ jobs:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
-        export FLEX_DATA_DIR=../../interactive/examples/movies/
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies ${GS_TEST_DIR}/flex/movies/movies_import.yaml \
-          ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ./engine_config_test.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
+        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
 

.gitmodules

@@ -9,3 +9,7 @@
 [submodule "learning_engine/graphlearn-for-pytorch"]
 	path = learning_engine/graphlearn-for-pytorch
 	url = https://github.com/alibaba/graphlearn-for-pytorch.git
+
+[submodule "flex/third_party/nlohmann-json"]
+	path = flex/third_party/nlohmann-json
+	url = https://github.com/nlohmann/json.git

docs/flex/interactive/configuration.md

@@ -44,6 +44,10 @@ compiler:
     - FilterIntoJoinRule
     - NotExistToAntiJoinRule
   query_timeout: 20000  # query timeout in milliseconds, default 20000
+http_service:
+  default_listen_address: localhost
+  admin_port: 7777
+  query_port: 10000
 ```
 
 
@@ -65,5 +69,8 @@ In this following table, we use the `.` notation to represent the hierarchy with
 | compiler.planner.rules.FilterMatchRule | N/A | An optimization rule that pushes filter (`Where`) conditions into the `Match` clause | 0.0.1 |
 | compiler.planner.rules.FilterIntoJoinRule | N/A | A native Calcite optimization rule that pushes filter conditions to the Join participants before performing the join | 0.0.1 |
 | compiler.planner.rules.NotMatchToAntiJoinRule | N/A | An optimization rule that transforms a "not exist" pattern into an anti-join operation  | 0.0.1 |
+| http_service.default_listen_address | localhost | The address for http service to bind | 0.0.2 |
+| http_service.admin_port | 7777 | The port for admin service to listen on | 0.0.2 |
+| http_service.query_port | 10000 | The port for query service to listen on, for stored procedure queries, user can directory submit queries to query_port without compiler involved | 0.0.2 |
 
 

docs/flex/interactive/dev_guide.md

@@ -1,6 +1,7 @@
 # GraphScope Interactive Development guide
 
 Users can develop C++ stored procedures and Cypher stored procedures based on `GraphScope Interactive`.
+Interactive also provides an Admin Service for managing graph data and stored procedures at runtime, and it can also retrieve the running status of the service.
 
 ```{toctree} arguments
 ---
@@ -9,4 +10,5 @@ maxdepth: 2
 ---
 development/cypher_procedure
 development/cpp_procedure
+development/admin_service
 ```