diff --git a/.github/workflows/flex.yml b/.github/workflows/flex.yml index 29dab8c68025..781529f37aa7 100644 --- a/.github/workflows/flex.yml +++ b/.github/workflows/flex.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6 + image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8 steps: - uses: actions/checkout@v3 @@ -44,6 +44,8 @@ jobs: env: HOME: /home/graphscope/ run: | + cd ${GITHUB_WORKSPACE}/ + git submodule update --init cd ${GITHUB_WORKSPACE}/flex mkdir build && cd build cmake .. && sudo make -j$(nproc) @@ -74,10 +76,10 @@ jobs: GLOG_v=10 ./bin/graph_loader ${SCHEMA_FILE} ${BULK_LOAD_FILE} /tmp/csr-data-dir/ - name: Test Graph Loading on type_test graph - env: + env: GS_TEST_DIR: ${{ github.workspace }}/gstest/ FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/type_test/ - run: | + run: | # remove modern graph indices rm -rf /tmp/csr-data-dir/ diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index e9a36dca3845..072d26ce67be 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6 + image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8 steps: - uses: actions/checkout@v3 @@ -68,6 +68,8 @@ jobs: GIE_HOME: ${{ github.workspace }}/interactive_engine/ HOME: /home/graphscope/ run: | + cd ${GITHUB_WORKSPACE}/ + git submodule update --init cd ${GITHUB_WORKSPACE}/flex mkdir build && cd build cmake .. && sudo make -j$(nproc) @@ -90,9 +92,27 @@ jobs: git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR} mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml + BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml + cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml + cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml + + # load graph + cd ${GITHUB_WORKSPACE}/flex/build + export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date + GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/ + export FLEX_DATA_DIR=../interactive/examples/movies + GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/indices/ + + - name: Test HQPS admin http service + env: + GS_TEST_DIR: ${{ github.workspace }}/gstest + FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph + run: + cd ${GITHUB_WORKSPACE}/flex/tests/hqps + bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR} - name: Sample Query test env: @@ -110,13 +130,12 @@ jobs: HOME : /home/graphscope/ INTERACTIVE_WORKSPACE: /tmp/interactive_workspace run: | - GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine cd ${GITHUB_WORKSPACE}/flex/bin for i in 1 2 3 4 5 6 7 8 9 10 11 12; do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" echo $cmd eval ${cmd} @@ -125,19 +144,19 @@ jobs: for i in 1 2 3 4 5 6 7 8 9 10 11 12; do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" echo $cmd eval ${cmd} done # test movie graph, 8,9,10 are not supported now - # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml for i in 1 2 3 4 5 6 7 11 12 13 14 15; do cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codgen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml" echo $cmd eval ${cmd} @@ -150,12 +169,11 @@ jobs: INTERACTIVE_WORKSPACE: /tmp/interactive_workspace run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ - export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date export ENGINE_TYPE=hiactor - # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to ldbc - sed -i 's/default_graph: movies/default_graph: ldbc/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml - bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \ - ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + # change the default_graph config in ./engine_config_test.yaml to ldbc + sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml + bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \ + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml - name: Run End-to-End cypher adhoc movie query test env: @@ -164,11 +182,10 @@ jobs: INTERACTIVE_WORKSPACE: /tmp/interactive_workspace run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ - export FLEX_DATA_DIR=../../interactive/examples/movies/ export ENGINE_TYPE=hiactor - # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml - bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies ${GS_TEST_DIR}/flex/movies/movies_import.yaml \ - ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml + # change the default_graph config in ./engine_config_test.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml + bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \ + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml diff --git a/.gitmodules b/.gitmodules index 1db7618bdbe0..490660f42055 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,7 @@ [submodule "learning_engine/graphlearn-for-pytorch"] path = learning_engine/graphlearn-for-pytorch url = https://github.com/alibaba/graphlearn-for-pytorch.git + +[submodule "flex/third_party/nlohmann-json"] + path = flex/third_party/nlohmann-json + url = https://github.com/nlohmann/json.git diff --git a/docs/flex/interactive/configuration.md b/docs/flex/interactive/configuration.md index 5f59c8b27e45..edcfb20554a2 100644 --- a/docs/flex/interactive/configuration.md +++ b/docs/flex/interactive/configuration.md @@ -44,6 +44,10 @@ compiler: - FilterIntoJoinRule - NotExistToAntiJoinRule query_timeout: 20000 # query timeout in milliseconds, default 20000 +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 10000 ``` @@ -65,5 +69,8 @@ In this following table, we use the `.` notation to represent the hierarchy with | compiler.planner.rules.FilterMatchRule | N/A | An optimization rule that pushes filter (`Where`) conditions into the `Match` clause | 0.0.1 | | compiler.planner.rules.FilterIntoJoinRule | N/A | A native Calcite optimization rule that pushes filter conditions to the Join participants before performing the join | 0.0.1 | | compiler.planner.rules.NotMatchToAntiJoinRule | N/A | An optimization rule that transforms a "not exist" pattern into an anti-join operation | 0.0.1 | +| http_service.default_listen_address | localhost | The address for http service to bind | 0.0.2 | +| http_service.admin_port | 7777 | The port for admin service to listen on | 0.0.2 | +| http_service.query_port | 10000 | The port for query service to listen on, for stored procedure queries, user can directory submit queries to query_port without compiler involved | 0.0.2 | diff --git a/docs/flex/interactive/dev_guide.md b/docs/flex/interactive/dev_guide.md index 66962775369f..0ef89c98c686 100644 --- a/docs/flex/interactive/dev_guide.md +++ b/docs/flex/interactive/dev_guide.md @@ -1,6 +1,7 @@ # GraphScope Interactive Development guide Users can develop C++ stored procedures and Cypher stored procedures based on `GraphScope Interactive`. +Interactive also provides an Admin Service for managing graph data and stored procedures at runtime, and it can also retrieve the running status of the service. ```{toctree} arguments --- @@ -9,4 +10,5 @@ maxdepth: 2 --- development/cypher_procedure development/cpp_procedure +development/admin_service ``` \ No newline at end of file diff --git a/docs/flex/interactive/development/admin_service.md b/docs/flex/interactive/development/admin_service.md new file mode 100644 index 000000000000..b6ca3919953c --- /dev/null +++ b/docs/flex/interactive/development/admin_service.md @@ -0,0 +1,1031 @@ +# GraphScope Interactive Admin Service Documentation + +## Introduction + +Welcome to the GraphScope Interactive Admin Service documentation. This guide is tailored for developers and administrators seeking to manage the Interactive service more efficiently. Here, we delve into the intricate workings of the RESTful HTTP interfaces provided by the Interactive Admin service, offering a comprehensive toolkit for real-time service management. This document is crucial for those looking to customize or enhance their GraphScope Interactive experience. + +## API Overview + +The table below provides an overview of the available APIs, categorized for ease of understanding: + + +| Category | API name | Method | URL | Explanation | +|---------------------|-----------------|--------|---------------------------------------|--------------------------------------------------------------------| +| GraphManagement | ListGraphs | GET | /v1/graph | Get all graphs in current interactive service, the schema for each graph is returned. | +| GraphManagement | GetGraphSchema | GET | /v1/graph/{graph_name}/schema | Get the schema for the specified graph. | +| GraphManagement | CreateGraph | POST | /v1/graph | Create an empty graph with the specified schema. | +| GraphManagement | DeleteGraph | DELETE | /v1/graph/{graph_name} | Delete the specified graph. | +| GraphManagement | ImportGraph | POST | /v1/graph/{graph_name}/dataloading | Import data to graph. | +| ProcedureManagement | CreateProcedure | POST | /v1/graph/{graph_name}/procedure | Create a new stored procedure bound to a graph. | +| ProcedureManagement | ShowProcedures | GET | /v1/graph/{graph_name}/procedure | Get all procedures bound to the specified graph. | +| ProcedureManagement | GetProcedure | GET | /v1/graph/{graph_name}/procedure/{procedure_name} | Get the metadata of the procedure. | +| ProcedureManagement | DeleteProcedure | DELETE | /v1/graph/{graph_name}/procedure/{procedure_name} | Delete the specified procedure. | +| ProcedureManagement | UpdateProcedure | PUT | /v1/graph/{graph_name}/procedure/{procedure_name} | Update some metadata for the specified procedure, i.e. update description, enable/disable. | +| ServiceManagement | StartService | POST | /v1/service/start | Start the service on the graph specified in request body. | +| ServiceManagement | ServiceStatus | GET | /v1/service/status | Get current service status. | +| SystemMetrics | SystemMetrics | GET | /v1/node/status | Get the system metrics of current host/pod, i.e. CPU usage, memory usages. | + + +## Detailed API Documentation + +For each API, the documentation will include a detailed description, request format, example curl command, expected response format and body, and status codes. Here's an example for one of the APIs: + + +### ListGraphs API (GraphManagement Category) + +#### Description +This API lists all graphs currently managed by the Interactive service, providing detailed schema information for each. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/graph` +- **Content-type**: `application/json` + +#### Curl Command Example +```bash +curl -X GET -H "Content-Type: application/json" "http://[host]/v1/graph" +``` + +#### Expected Response +- **Format**: `application/json` +- **Body**: +```json +[ + { + "name": "modern_graph", + "schema": { + "vertex_types": [ + { + "type_id": "0", + "type_name": "person", + "properties": [ + { + "property_id": "0", + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": "1", + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": "2", + "property_name": "age", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "id" + ] + }, + { + "type_id": "1", + "type_name": "software", + "properties": [ + { + "property_id": "0", + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": "1", + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": "2", + "property_name": "lang", + "property_type": { + "primitive_type": "DT_STRING" + } + } + ], + "primary_keys": [ + "id" + ] + } + ], + "edge_types": [ + { + "type_id": "0", + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": "0", + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + }, + { + "type_id": "1", + "type_name": "created", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "software", + "relation": "ONE_TO_MANY", + } + ], + "properties": [ + { + "property_id": "0", + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + } + ] + } + } +] +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. + +### CreateGraph (GraphManagement Category) + +#### Description +This API create a new graph according to the specified schema in request body. + + +#### HTTP Request +- **Method**: POST +- **Endpoint**: `/v1/graph` +- **Content-type**: `application/json` +- **Body**: +```json +{ + "name": "modern", + "schema": { + "vertex_types": [ + { + "type_id": 0, + "type_name": "person", + "properties": [ + { + "property_id": 0, + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": 1, + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": 2, + "property_name": "age", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "id" + ] + }, + { + "type_id": 1, + "type_name": "software", + "properties": [ + { + "property_id": 0, + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": 1, + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": 2, + "property_name": "lang", + "property_type": { + "primitive_type": "DT_STRING" + } + } + ], + "primary_keys": [ + "id" + ] + } + ], + "edge_types": [ + { + "type_id": 0, + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + }, + { + "type_id": 1, + "type_name": "created", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "software", + "relation": "ONE_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + } + ] + } +} +``` + +#### Curl Command Example +```bash +curl -X POST -H "Content-Type: application/json" -d @path/to/yourfile.json "http://[host]/v1/graph" +``` + +#### Expected Response + +- **Format**: `application/json` +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph not found + +### DeleteGraph (GraphManagement Category) + +#### Description +Delete a graph by name, including schema, indices and stored procedures. + +#### HTTP Request +- **Method**: DELETE +- **Endpoint**: `/v1/graph/{graph_name}` +- **Content-type**: `application/json` + + +#### Curl Command Example +```bash +curl -X DELETE -H "Content-Type: application/json" "http://[host]/v1/graph/{graph_name}" +``` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. + +### GetGraphSchema (GraphManagement Category) + +#### Description +Get the schema for the specified graph. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/graph/{graph_name}` +- **Content-type**: `application/json` + +#### Curl Command Example +```bash +curl -X GET -H "Content-Type: application/json" "http://[host]/v1/graph/{graph_name}" +``` + + +#### Expected Response +- **Format**: `application/json` +- **Body**: +```json +{ + "name": "modern", + "schema": { + "vertex_types": [ + { + "type_id": 0, + "type_name": "person", + "properties": [ + { + "property_id": 0, + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": 1, + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": 2, + "property_name": "age", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "id" + ] + }, + { + "type_id": 1, + "type_name": "software", + "properties": [ + { + "property_id": 0, + "property_name": "id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_id": 1, + "property_name": "name", + "property_type": { + "primitive_type": "DT_STRING" + } + }, + { + "property_id": 2, + "property_name": "lang", + "property_type": { + "primitive_type": "DT_STRING" + } + } + ], + "primary_keys": [ + "id" + ] + } + ], + "edge_types": [ + { + "type_id": 0, + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + }, + { + "type_id": 1, + "type_name": "created", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "software", + "relation": "ONE_TO_MANY", + } + ], + "properties": [ + { + "property_id": 0, + "property_name": "weight", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ] + } + ] + } +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph not found + +### ImportGraph (GraphManagement Category) + +#### Description + +Import data to empty graph. + +#### HTTP Request + +- **Method**: POST +- **Endpoint**: `/v1/graph/{graph_name}/dataloading` +- **Content-type**: `application/json` +- **Body**: +```json +{ + "graph": "graph", + "loading_config": { + "data_source": { + "scheme": "file" + "location": {path_to_file}, + }, + "format": { + "metadata": { + "key": "metadata" + }, + "type": "type" + }, + "import_option": "init" + }, + "edge_mappings": [ + { + "column_mappings": [ + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + }, + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "name" + } + }, + { + "column": { + "index": 0, + "name": "name" + } + } + ], + "inputs": [ + "inputs", + "inputs" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "name" + } + }, + { + "column": { + "index": 0, + "name": "name" + } + } + ], + "type_triplet": { + "destination_vertex": "destination_vertex", + "edge": "edge", + "source_vertex": "source_vertex" + } + }, + { + "column_mappings": [ + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + }, + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "name" + } + }, + { + "column": { + "index": 0, + "name": "name" + } + } + ], + "inputs": [ + "inputs", + "inputs" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "name" + } + }, + { + "column": { + "index": 0, + "name": "name" + } + } + ], + "type_triplet": { + "destination_vertex": "destination_vertex", + "edge": "edge", + "source_vertex": "source_vertex" + } + } + ], + "vertex_mappings": [ + { + "column_mappings": [ + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + }, + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + } + ], + "inputs": [ + "inputs", + "inputs" + ], + "type_name": "type_name" + }, + { + "column_mappings": [ + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + }, + { + "column": { + "index": 0, + "name": "name" + }, + "property": "property" + } + ], + "inputs": [ + "inputs", + "inputs" + ], + "type_name": "type_name" + } + ] +} +``` + +#### Curl Command Example +```bash +curl -X POST -H "Content-Type: application/json" -d @path/to/json "http://[host]/v1/graph/{graph_name}/dataloading" +``` + +#### Expected Response +- **Format**: `application/json` +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph not found. + + +### CreateProcedure (ProcedureManagement Category) + +#### Description + +Create a new stored procedure. + +#### HTTP Request +- **Method**: POST +- **Endpoint**: `/v1/graph/{graph_name}/procedure` +- **Content-type**: `application/json` +- **Body**: +```json +{ + "bound_graph": "bound_graph", + "description": "description", + "enable": true, + "name": "name", + "query" : "MATCH(n) return COUNT(n);", + "type": "cypher" +} +``` + +#### Curl Command Example +```bash +curl -X POST -H "Content-Type: application/json" -d @/path/to/json "http://[host]/v1/graph/{graph_name}/procedure" +``` + + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not FOund`: Graph not found. + +### ListAllProcedure (ProcedureManagement Category) + +#### Description + +List all procedures bound to a graph. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/graph/{graph_name}/procedure` +- **Content-type**: `application/json` + +#### Curl Command Example +```bash +curl -X GET -H "Content-Type: application/json" "http://[host]/v1/graph/{graph_name}/procedure" +``` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +[ + { + "bound_graph": "bound_graph", + "description": "description", + "enable": true, + "name": "name", + "params": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "query": "query", + "returns": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "type": "cpp" + } +] +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph not found + + +### GetProcedure (ProcedureManagement Category) + +#### Description + +Get a single procedure's metadata. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/graph/{graph_name}/procedure/{procedure_name}` +- **Content-type**: `application/json` + +#### Curl Command Example +```bash +curl -X GET -H "Content-Type: application/json" "http://[host]/v1/graph/{graph_name}/procedure/{procedure_name}" +``` + + +#### Expected Response +- **Format**: application/json +- **Body**: +```json + +{ + "bound_graph": "bound_graph", + "description": "description", + "enable": true, + "name": "name", + "params": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "query": "query", + "returns": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "type": "cpp" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. + + +### UpdateProcedure (ProcedureManagement Category) + +#### Description + +Update a procedure's metadata, enable/disable status, description. The procedure's name can not be modified. + + +#### HTTP Request +- **Method**: PUT +- **Endpoint**: `/v1/graph/{graph_name}/procedure/{procedure_name}` +- **Content-type**: `application/json` +- **Body**: +```json +{ + "description": "description", + "enable": true, +} +``` + +#### Curl Command Example +```bash +curl -X PUT -H "Content-Type: application/json" -d @/path/to/json "http://[host]//v1/graph/{graph_name}/procedure/{procedure_name}" +``` + +#### Expected Response +- **Format**: `application/json` +- **Body**: +```json +{ + "bound_graph": "bound_graph", + "description": "description", + "enable": true, + "name": "name", + "params": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "query": "query", + "returns": [ + { + "name": "name", + "type": "type" + }, + { + "name": "name", + "type": "type" + } + ], + "type": "cpp" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph or procedure not found. + +### DeleteProcedure (ProcedureManagement Category) + +#### Description + +Delete a procedure bound to the graph. + +#### HTTP Request + +- **Method**: DELETE +- **Endpoint**: `/v1/graph/{graph_name}/procedure/{procedure_name}` +- **Content-type**: `application/json` + + +#### Curl Command Example +```bash +curl -X DELETE -H "Content-Type: application/json" "http://[host]/v1/graph/{graph_name}/procedure/{procedure_name}" +``` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. +- `404 Not Found`: Graph or procedure not found. + +### StartService (ServiceManagement Category) + +#### Description + +Start the query service on a graph. + +#### HTTP Request +- **Method**: POST +- **Endpoint**: `/v1/service/start` +- **Content-type**: `application/json` +- **Body**: +```json +{ + "graph_name": "modern_graph" +} +``` + +#### Curl Command Example +```bash +curl -X POST -H "Content-Type: application/json" "http://[host]/v1/service/start" +``` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "message": "message" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. + +### ServiceStatus + +#### Description + +Get the status of current service. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/service/status` +- **Content-type**: `application/json` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "graph_name": "graph_name", + "query_port": 6, + "status": "running" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. + + +### SystemMetrics (NodeMetrics Category) + +#### Description + +Get node status. + +#### HTTP Request +- **Method**: GET +- **Endpoint**: `/v1/node/status` +- **Content-type**: `application/json` + +#### Curl Command Example +```bash +curl -X GET -H "Content-Type: application/json" "http://[host]/v1/node/status" +``` + +#### Expected Response +- **Format**: application/json +- **Body**: +```json +{ + "cpu_usage": "0.2", + "memory_usage": "0.3" +} +``` + +#### Status Codes +- `200 OK`: Request successful. +- `500 Internal Error`: Server internal Error. \ No newline at end of file diff --git a/flex/.devcontainer.json b/flex/.devcontainer.json index 6118d0fbdae8..d1b8c725dca5 100644 --- a/flex/.devcontainer.json +++ b/flex/.devcontainer.json @@ -3,7 +3,7 @@ { "name": "GraphScope", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6", + "image": "registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8", // Features to add to the dev container. More info: https://containers.dev/features. "features": { "ghcr.io/devcontainers/features/common-utils:2": { diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 591e02d836e2..ab341850e2d5 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -96,6 +96,13 @@ if (BUILD_DOC) endif(DOXYGEN_FOUND) endif() +# Check whether ${CMAKE_SOURCE_DIR}/third_party/single_include/nlohmann/json.hpp exists +if (NOT EXISTS ${CMAKE_SOURCE_DIR}/third_party/nlohmann-json/single_include/nlohmann/json.hpp) + message(FATAL_ERROR "${CMAKE_SOURCE_DIR}/third_party/nlohmann-json/single_include/nlohmann/json.hpp not found, " + "please run `git submodule update --init` to download third_party") +endif() +include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/third_party/single_include) + add_subdirectory(utils) add_subdirectory(codegen) add_subdirectory(storages) diff --git a/flex/bin/CMakeLists.txt b/flex/bin/CMakeLists.txt index 1ae40148e851..2509ae0284d0 100644 --- a/flex/bin/CMakeLists.txt +++ b/flex/bin/CMakeLists.txt @@ -46,10 +46,10 @@ install(TARGETS graph_loader if(BUILD_HQPS) if(Hiactor_FOUND) - add_executable(sync_server sync_server.cc) - target_link_libraries(sync_server flex_utils flex_graph_db flex_server hqps_plan_proto ${GLOG_LIBRARIES}) + add_executable(interactive_server interactive_server.cc) + target_link_libraries(interactive_server flex_utils flex_graph_db flex_server hqps_plan_proto flex_utils ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES}) - install(TARGETS sync_server + install(TARGETS interactive_server RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib) diff --git a/flex/bin/graph_loader.cc b/flex/bin/graph_loader.cc index e0280a836b03..7b5991b0386e 100644 --- a/flex/bin/graph_loader.cc +++ b/flex/bin/graph_loader.cc @@ -43,7 +43,7 @@ int main(int argc, char** argv) { auto schema = gs::Schema::LoadFromYaml(schema_file); auto bulk_load_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_config_path); + gs::LoadingConfig::ParseFromYamlFile(schema, bulk_load_config_path); db.Init(schema, bulk_load_config, data_path, thread_num); t0 += grape::GetCurrentTime(); diff --git a/flex/bin/interactive_server.cc b/flex/bin/interactive_server.cc new file mode 100644 index 000000000000..52e210a7ada6 --- /dev/null +++ b/flex/bin/interactive_server.cc @@ -0,0 +1,333 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "stdlib.h" + +#include "flex/engines/http_server/codegen_proxy.h" +#include "flex/engines/http_server/service/hqps_service.h" +#include "flex/engines/http_server/workdir_manipulator.h" +#include "flex/storages/rt_mutable_graph/loading_config.h" +#include "flex/utils/service_utils.h" + +#include +#include + +#include + +namespace bpo = boost::program_options; + +namespace gs { +static constexpr const uint32_t DEFAULT_SHARD_NUM = 1; +static constexpr const uint32_t DEFAULT_QUERY_PORT = 10000; +static constexpr const uint32_t DEFAULT_ADMIN_PORT = 7777; + +std::string parse_codegen_dir(const bpo::variables_map& vm) { + std::string codegen_dir; + + if (vm.count("codegen-dir") == 0) { + LOG(INFO) << "codegen-dir is not specified"; + codegen_dir = server::CodegenProxy::DEFAULT_CODEGEN_DIR; + } else { + codegen_dir = vm["codegen-dir"].as(); + } + // clear codegen dir + if (std::filesystem::exists(codegen_dir)) { + LOG(INFO) << "codegen dir exists, clear directory"; + std::filesystem::remove_all(codegen_dir); + } else { + // create codegen_dir + LOG(INFO) << "codegen dir not exists, create directory"; + std::filesystem::create_directory(codegen_dir); + } + return codegen_dir; +} + +// parse from yaml +std::tuple parse_from_server_config( + const std::string& server_config_path) { + YAML::Node config = YAML::LoadFile(server_config_path); + uint32_t shard_num = DEFAULT_SHARD_NUM; + uint32_t query_port = DEFAULT_QUERY_PORT; + uint32_t admin_port = DEFAULT_ADMIN_PORT; + auto engine_node = config["compute_engine"]; + if (engine_node) { + auto engine_type = engine_node["type"]; + if (engine_type) { + auto engine_type_str = engine_type.as(); + if (engine_type_str != "hiactor" && engine_type_str != "Hiactor") { + LOG(FATAL) << "compute_engine type should be hiactor, found: " + << engine_type_str; + } + } + auto shard_num_node = engine_node["thread_num_per_worker"]; + if (shard_num_node) { + shard_num = shard_num_node.as(); + } else { + LOG(INFO) << "shard_num not found, use default value " + << DEFAULT_SHARD_NUM; + } + } else { + LOG(FATAL) << "Fail to find compute_engine configuration"; + } + auto http_service_node = config["http_service"]; + if (http_service_node) { + auto query_port_node = http_service_node["query_port"]; + if (query_port_node) { + query_port = query_port_node.as(); + } else { + LOG(INFO) << "query_port not found, use default value " + << DEFAULT_QUERY_PORT; + } + auto admin_port_node = http_service_node["admin_port"]; + if (admin_port_node) { + admin_port = admin_port_node.as(); + } else { + LOG(INFO) << "admin_port not found, use default value " + << DEFAULT_ADMIN_PORT; + } + } else { + LOG(FATAL) << "Fail to find http_service configuration"; + } + return std::make_tuple(shard_num, admin_port, query_port); +} + +void init_codegen_proxy(const bpo::variables_map& vm, + const std::string& graph_schema_file, + const std::string& engine_config_file) { + std::string codegen_dir = parse_codegen_dir(vm); + std::string codegen_bin; + if (vm.count("codegen-bin") == 0) { + LOG(INFO) << "codegen-bin is not specified"; + codegen_bin = find_codegen_bin(); + } else { + LOG(INFO) << "codegen-bin is specified"; + codegen_bin = vm["codegen-bin"].as(); + if (!std::filesystem::exists(codegen_bin)) { + LOG(FATAL) << "codegen bin not exists: " << codegen_bin; + } + } + server::CodegenProxy::get().Init(codegen_dir, codegen_bin, engine_config_file, + graph_schema_file); +} + +void parse_args(bpo::variables_map& vm, int32_t& admin_port, + int32_t& query_port, std::string& workspace) { + if (vm.count("admin-port")) { + admin_port = vm["admin-port"].as(); + } + if (vm.count("query-port")) { + query_port = vm["query-port"].as(); + } + if (vm.count("workspace")) { + workspace = vm["workspace"].as(); + } +} + +void initWorkspace(const std::string workspace, int32_t thread_num) { + // If workspace directory not exists, create. + + auto default_graph = server::HQPSService::DEFAULT_GRAPH_NAME; + if (!std::filesystem::exists(workspace)) { + std::filesystem::create_directory(workspace); + } + // Create subdirectories + std::filesystem::create_directory(workspace + "/" + + server::WorkDirManipulator::DATA_DIR_NAME); + std::filesystem::create_directory(workspace + "/conf"); + std::filesystem::create_directory(workspace + "/log"); + + LOG(INFO) << "Finish creating workspace directory " << workspace; + // Get current executable path + + std::string exe_dir = gs::get_current_dir(); + LOG(INFO) << "Executable directory: " << exe_dir; + std::string interactive_home = exe_dir + "/../../interactive"; + // check exists + if (!std::filesystem::exists(interactive_home)) { + LOG(FATAL) << "Interactive home directory " << interactive_home + << " not exists, exit."; + } + LOG(INFO) << "Interactive home: " << interactive_home; + // copy conf files + std::filesystem::copy_file( + interactive_home + "/conf/" + + server::WorkDirManipulator::CONF_ENGINE_CONFIG_FILE_NAME, + workspace + "/conf/" + + server::WorkDirManipulator::CONF_ENGINE_CONFIG_FILE_NAME, + std::filesystem::copy_options::overwrite_existing); + std::filesystem::copy_file(interactive_home + "/conf/interactive.yaml", + workspace + "/conf/interactive.yaml", + std::filesystem::copy_options::overwrite_existing); + // create modern_graph directory + std::filesystem::create_directory(workspace + "/data/" + default_graph); + auto bulk_loading_file = + interactive_home + "/examples/" + default_graph + "/bulk_load.yaml"; + // copy modern_graph files + if (!std::filesystem::exists(bulk_loading_file)) { + LOG(FATAL) << "Bulk loading file " << bulk_loading_file + << " not exists, exit."; + } + + std::filesystem::copy_file( + interactive_home + "/examples/modern_graph/modern_graph.yaml", + workspace + "/data/modern_graph/graph.yaml", + std::filesystem::copy_options::overwrite_existing); + + server::WorkDirManipulator::SetWorkspace(workspace); + + // gs::run_graph_loading(schema_path, bulk_loading_file, data_dir); + auto schema_path = + server::WorkDirManipulator::GetGraphSchemaPath(default_graph); + + auto res = server::WorkDirManipulator::LoadGraph(bulk_loading_file, + default_graph, 1); + if (!res.ok()) { + LOG(FATAL) << "Fail to load graph: " << res.status().error_message(); + } + + VLOG(1) << "Finish init workspace"; + + auto& db = gs::GraphDB::get(); + + gs::Schema schema = gs::Schema::LoadFromYaml(schema_path); + auto data_dir_res = + server::WorkDirManipulator::GetDataDirectory(default_graph); + if (!data_dir_res.ok()) { + LOG(FATAL) << "Fail to get data directory for default graph: " + << data_dir_res.status().error_message(); + } + auto data_dir = data_dir_res.value(); + db.Close(); + if (!db.Open(schema, data_dir, thread_num).ok()) { + LOG(FATAL) << "Fail to load graph from data directory: " << data_dir; + } + LOG(INFO) << "Successfully init graph db for default graph: " + << server::HQPSService::DEFAULT_GRAPH_NAME; + + server::WorkDirManipulator::SetRunningGraph( + server::HQPSService::DEFAULT_GRAPH_NAME); +} + +} // namespace gs + +/** + * The main entrance for InteractiveServer. + */ +int main(int argc, char** argv) { + bpo::options_description desc("Usage:"); + desc.add_options()("help,h", "Display help messages")( + "enable-admin-service,e", bpo::value()->default_value(false), + "whether or not to start admin service")("server-config,c", + bpo::value(), + "path to server config yaml")( + "codegen-dir,d", + bpo::value()->default_value("/tmp/codegen/"), + "codegen working directory")("shard-num,s", bpo::value(), + "shard number")( + "workspace,w", + bpo::value()->default_value("/tmp/workspace/"), + "directory to interactive workspace")( + "graph-config,g", bpo::value(), "graph schema config file")( + "data-path,a", bpo::value(), "data directory path")( + "open-thread-resource-pool", bpo::value()->default_value(true), + "open thread resource pool")("worker-thread-number", + bpo::value()->default_value(2), + "worker thread number"); + + setenv("TZ", "Asia/Shanghai", 1); + tzset(); + + bpo::variables_map vm; + bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm); + bpo::notify(vm); + + if (vm.count("help")) { + std::cout << desc << std::endl; + return 0; + } + + //// declare vars + int32_t shard_num = 1; + int32_t admin_port = gs::DEFAULT_ADMIN_PORT; + int32_t query_port = gs::DEFAULT_QUERY_PORT; + bool start_admin_service; + + if (vm.count("shard-num")) { + shard_num = vm["shard-num"].as(); + } + VLOG(10) << "Set shard num to " << shard_num; + start_admin_service = vm["enable-admin-service"].as(); + + std::string workspace; + gs::parse_args(vm, admin_port, query_port, workspace); + auto& db = gs::GraphDB::get(); + + if (start_admin_service) { + // When start admin service, we need a workspace to put all the meta data + // and graph indices. We will initiate the query service with default graph. + if (vm.count("graph-config") || vm.count("data-path")) { + LOG(FATAL) << "To start admin service, graph-config and " + "data-path should NOT be specified"; + } + gs::initWorkspace(workspace, shard_num); // the default graph is loaded. + LOG(INFO) << "Finish init workspace"; + + server::HQPSService::get().init(shard_num, admin_port, query_port, false, + vm["open-thread-resource-pool"].as(), + vm["worker-thread-number"].as()); + server::HQPSService::get().run_and_wait_for_exit(); + } else { + LOG(INFO) << "Start query service only"; + std::string graph_schema_path, data_path; + if (!vm.count("server-config")) { + LOG(FATAL) << "server-config is needed"; + } + auto engine_config_file = vm["server-config"].as(); + // When only starting query service. + std::tie(shard_num, admin_port, query_port) = + gs::parse_from_server_config(engine_config_file); + + // init graph + if (!vm.count("graph-config")) { + LOG(ERROR) << "graph-config is required"; + return -1; + } + graph_schema_path = vm["graph-config"].as(); + if (!vm.count("data-path")) { + LOG(ERROR) << "data-path is required"; + return -1; + } + data_path = vm["data-path"].as(); + + auto schema = gs::Schema::LoadFromYaml(graph_schema_path); + // Ths schema is loaded just to get the plugin dir and plugin list + gs::init_codegen_proxy(vm, graph_schema_path, engine_config_file); + db.Close(); + auto load_res = db.Open(schema, data_path, shard_num); + if (!load_res.ok()) { + LOG(FATAL) << "Failed to load graph from data directory: " + << load_res.status().error_message(); + } + + server::HQPSService::get().init(shard_num, query_port, false, + vm["open-thread-resource-pool"].as(), + vm["worker-thread-number"].as()); + server::HQPSService::get().run_and_wait_for_exit(); + } + + return 0; +} diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh index 994ce3ddf1e4..3afc76828d47 100755 --- a/flex/bin/load_plan_and_gen.sh +++ b/flex/bin/load_plan_and_gen.sh @@ -316,8 +316,7 @@ compile_hqps_so() { fi # check output_dir doesn't contains output_so_name if [ -f ${dst_so_path} ]; then - err "Output dir ${output_dir} already contains ${procedure_name}.so, please remove it first." - exit 1 + emph "Output dir ${output_dir} already contains ${procedure_name}.so,overriding it." fi cp ${output_so_path} ${output_dir} #check dst_so_path exists @@ -438,8 +437,7 @@ compile_pegasus_so() { fi # check output_dir doesn't contains output_so_name if [ -f ${dst_so_path} ]; then - err "Output dir ${output_dir} already contains ${query_name}.so, please remove it first." - exit 1 + err "Output dir ${output_dir} already contains ${query_name}.so, overriding it." fi cp ${output_so_path} ${output_dir} #check dst_so_path exists diff --git a/flex/bin/rt_bench.cc b/flex/bin/rt_bench.cc index 94ea8d70a8c6..ab1b16d42e4b 100644 --- a/flex/bin/rt_bench.cc +++ b/flex/bin/rt_bench.cc @@ -23,8 +23,8 @@ #include #include "flex/engines/graph_db/database/graph_db.h" #include "flex/engines/http_server/executor_group.actg.h" -#include "flex/engines/http_server/generated/executor_ref.act.autogen.h" -#include "flex/engines/http_server/graph_db_service.h" +#include "flex/engines/http_server/generated/actor/executor_ref.act.autogen.h" +#include "flex/engines/http_server/service/graph_db_service.h" namespace bpo = boost::program_options; using namespace std::chrono_literals; @@ -209,7 +209,7 @@ int main(int argc, char** argv) { auto schema = gs::Schema::LoadFromYaml(graph_schema_path); auto loading_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_config_path); + gs::LoadingConfig::ParseFromYamlFile(schema, bulk_load_config_path); db.Init(schema, loading_config, data_path, shard_num); t0 += grape::GetCurrentTime(); diff --git a/flex/bin/rt_server.cc b/flex/bin/rt_server.cc index b8839a729300..92d2b7ef4597 100644 --- a/flex/bin/rt_server.cc +++ b/flex/bin/rt_server.cc @@ -16,8 +16,8 @@ #include "grape/util.h" #include "flex/engines/graph_db/database/graph_db.h" -#include "flex/engines/http_server/graph_db_service.h" #include "flex/engines/http_server/options.h" +#include "flex/engines/http_server/service/graph_db_service.h" #include #include @@ -84,7 +84,7 @@ int main(int argc, char** argv) { auto schema = gs::Schema::LoadFromYaml(graph_schema_path); auto loading_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_config_path); + gs::LoadingConfig::ParseFromYamlFile(schema, bulk_load_config_path); db.Init(schema, loading_config, data_path, shard_num); t0 += grape::GetCurrentTime(); diff --git a/flex/bin/sync_server.cc b/flex/bin/sync_server.cc deleted file mode 100644 index 5d9ca2adf32c..000000000000 --- a/flex/bin/sync_server.cc +++ /dev/null @@ -1,266 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "stdlib.h" - -#include "flex/engines/http_server/hqps_service.h" - -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" -#include "flex/engines/http_server/codegen_proxy.h" -#include "flex/engines/http_server/stored_procedure.h" -#include "flex/storages/rt_mutable_graph/loading_config.h" - -#include -#include - -namespace bpo = boost::program_options; - -namespace gs { -static constexpr const char* CODEGEN_BIN = "load_plan_and_gen.sh"; -static constexpr const uint32_t DEFAULT_SHARD_NUM = 1; -static constexpr const uint32_t DEFAULT_HTTP_PORT = 10000; -static constexpr const char* DEFAULT_CODEGEN_DIR = "/tmp/codegen/"; -std::string find_codegen_bin() { - // first check whether flex_home env exists - std::string flex_home; - std::string codegen_bin; - char* flex_home_char = getenv("FLEX_HOME"); - if (flex_home_char == nullptr) { - // infer flex_home from current binary' directory - char* bin_path = realpath("/proc/self/exe", NULL); - std::string bin_path_str(bin_path); - // flex home should be bin_path/../../ - std::string flex_home_str = - bin_path_str.substr(0, bin_path_str.find_last_of("/")); - // usr/loca/bin/ - flex_home_str = flex_home_str.substr(0, flex_home_str.find_last_of("/")); - // usr/local/ - - LOG(INFO) << "infer flex_home as installed, flex_home: " << flex_home_str; - // check codege_bin_path exists - codegen_bin = flex_home_str + "/bin/" + CODEGEN_BIN; - // if flex_home exists, return flex_home - if (std::filesystem::exists(codegen_bin)) { - return codegen_bin; - } else { - // if not found, try as if it is in build directory - // flex/build/ - flex_home_str = flex_home_str.substr(0, flex_home_str.find_last_of("/")); - // flex/ - LOG(INFO) << "infer flex_home as build, flex_home: " << flex_home_str; - codegen_bin = flex_home_str + "/bin/" + CODEGEN_BIN; - if (std::filesystem::exists(codegen_bin)) { - return codegen_bin; - } else { - LOG(FATAL) << "codegen bin not exists: "; - return ""; - } - } - } else { - flex_home = std::string(flex_home_char); - LOG(INFO) << "flex_home env exists, flex_home: " << flex_home; - codegen_bin = flex_home + "/bin/" + CODEGEN_BIN; - if (std::filesystem::exists(codegen_bin)) { - return codegen_bin; - } else { - LOG(FATAL) << "codegen bin not exists: "; - return ""; - } - } -} - -std::string parse_codegen_dir(const bpo::variables_map& vm) { - std::string codegen_dir; - - if (vm.count("codegen-dir") == 0) { - LOG(INFO) << "codegen-dir is not specified"; - codegen_dir = DEFAULT_CODEGEN_DIR; - } else { - codegen_dir = vm["codegen-dir"].as(); - } - // clear codegen dir - if (std::filesystem::exists(codegen_dir)) { - LOG(INFO) << "codegen dir exists, clear directory"; - std::filesystem::remove_all(codegen_dir); - } else { - // create codegen_dir - LOG(INFO) << "codegen dir not exists, create directory"; - std::filesystem::create_directory(codegen_dir); - } - return codegen_dir; -} - -// parse from yaml -std::tuple parse_from_server_config( - const std::string& server_config_path) { - YAML::Node config = YAML::LoadFile(server_config_path); - uint32_t shard_num = DEFAULT_SHARD_NUM; - uint32_t http_port = DEFAULT_HTTP_PORT; - auto engine_node = config["compute_engine"]; - if (engine_node) { - auto engine_type = engine_node["type"]; - if (engine_type) { - auto engine_type_str = engine_type.as(); - if (engine_type_str != "hiactor" && engine_type_str != "Hiactor") { - LOG(FATAL) << "compute_engine type should be hiactor, found: " - << engine_type_str; - } - } - auto shard_num_node = engine_node["shard_num"]; - if (shard_num_node) { - shard_num = shard_num_node.as(); - } else { - LOG(INFO) << "shard_num not found, use default value " - << DEFAULT_SHARD_NUM; - } - auto host_node = engine_node["hosts"]; - if (host_node) { - // host node is a list - if (host_node.IsSequence()) { - CHECK(host_node.size() == 1) - << "only support one host in compute_engine configuration"; - auto host_str = host_node[0].as(); - auto port_pos = host_str.find(":"); - if (port_pos != std::string::npos) { - http_port = std::stoi(host_str.substr(port_pos + 1)); - } else { - LOG(FATAL) << "host_node not found, use default value "; - } - } - } else { - LOG(INFO) << "host_node not found, use default value " - << DEFAULT_HTTP_PORT; - } - return std::make_tuple(shard_num, http_port); - } else { - LOG(FATAL) << "Fail to find compute_engine configuration"; - } -} - -void init_codegen_proxy(const bpo::variables_map& vm, - const std::string& graph_schema_file, - const std::string& engine_config_file) { - std::string codegen_dir = parse_codegen_dir(vm); - std::string codegen_bin; - if (vm.count("codegen-bin") == 0) { - LOG(INFO) << "codegen-bin is not specified"; - codegen_bin = find_codegen_bin(); - } else { - LOG(INFO) << "codegen-bin is specified"; - codegen_bin = vm["codegen-bin"].as(); - if (!std::filesystem::exists(codegen_bin)) { - LOG(FATAL) << "codegen bin not exists: " << codegen_bin; - } - } - server::CodegenProxy::get().Init(codegen_dir, codegen_bin, engine_config_file, - graph_schema_file); -} -} // namespace gs - -int main(int argc, char** argv) { - bpo::options_description desc("Usage:"); - desc.add_options()("help,h", "Display help messages")( - "server-config,c", bpo::value(), - "path to server config yaml")( - "codegen-dir,d", - bpo::value()->default_value("/tmp/codegen/"), - "codegen working directory")("codegen-bin,b", bpo::value(), - "codegen binary path")( - "graph-config,g", bpo::value(), "graph schema config file")( - "data-path,a", bpo::value(), "data directory path")( - "bulk-load,l", bpo::value(), "bulk-load config file")( - "open-thread-resource-pool", bpo::value()->default_value(true), - "open thread resource pool")("worker-thread-number", - bpo::value()->default_value(2), - "worker thread number"); - - setenv("TZ", "Asia/Shanghai", 1); - tzset(); - - bpo::variables_map vm; - bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm); - bpo::notify(vm); - - if (vm.count("help")) { - std::cout << desc << std::endl; - return 0; - } - //// declare vars - uint32_t shard_num; - uint32_t http_port; - std::string graph_schema_path; - std::string data_path; - std::string bulk_load_config_path; - std::string plugin_dir; - std::string server_config_path; - - if (vm.count("server-config") != 0) { - server_config_path = vm["server-config"].as(); - // check file exists - if (!std::filesystem::exists(server_config_path)) { - LOG(ERROR) << "server-config not exists: " << server_config_path; - return 0; - } - std::tie(shard_num, http_port) = - gs::parse_from_server_config(server_config_path); - LOG(INFO) << "shard_num: " << shard_num << ", http_port: " << http_port; - } else { - LOG(FATAL) << "server-config is needed"; - } - - // init graph - if (!vm.count("graph-config")) { - LOG(ERROR) << "graph-config is required"; - return -1; - } - graph_schema_path = vm["graph-config"].as(); - if (!vm.count("data-path")) { - LOG(ERROR) << "data-path is required"; - return -1; - } - data_path = vm["data-path"].as(); - if (vm.count("bulk-load")) { - bulk_load_config_path = vm["bulk-load"].as(); - } - - double t0 = -grape::GetCurrentTime(); - auto& db = gs::GraphDB::get(); - - auto schema = gs::Schema::LoadFromYaml(graph_schema_path); - auto loading_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_config_path); - db.Init(schema, loading_config, data_path, shard_num); - - t0 += grape::GetCurrentTime(); - - LOG(INFO) << "Finished loading graph, elapsed " << t0 << " s"; - - // loading plugin - if (!schema.GetPluginDir().empty() && !schema.GetPluginsList().empty()) { - server::StoredProcedureManager::get().LoadFromPluginDir( - schema.GetPluginDir(), schema.GetPluginsList()); - } - - gs::init_codegen_proxy(vm, graph_schema_path, server_config_path); - - server::HQPSService::get().init(shard_num, http_port, false, - vm["open-thread-resource-pool"].as(), - vm["worker-thread-number"].as()); - server::HQPSService::get().run_and_wait_for_exit(); - - return 0; -} diff --git a/flex/codegen/README.md b/flex/codegen/README.md index bb71703d7158..03735d49a2b1 100644 --- a/flex/codegen/README.md +++ b/flex/codegen/README.md @@ -26,21 +26,26 @@ struct Expression1{ //2. Query class -class Query0 : public HqpsAppBase { +class Query0 : public AppBase { public: - results::CollectiveResults Query(const MutableCSRInterface& graph, - int64_t time_stamp) const override { - - + bool Query(Decoder& input, Encoder& output) override { + //... } }; } // namespace gs // 3. Create and delete handler for query extern "C" { -void* CreateApp(gs::GraphStoreType store_type) { +void* CreateApp(gs::GraphDBSession& db) { + Query0* app = new Query0(db); + return static_cast(app); } -void DeleteApp(void* app, gs::GraphStoreType store_type) { + +void DeleteApp(void* app) { + if (app){ + Query0* casted = static_cast(app); + delete casted; + } } } ``` diff --git a/flex/codegen/src/building_context.h b/flex/codegen/src/building_context.h index c6f46eda264d..628c043d0c2d 100644 --- a/flex/codegen/src/building_context.h +++ b/flex/codegen/src/building_context.h @@ -41,7 +41,7 @@ static constexpr const char* PATH_OPT_NAME = "path_opt"; static constexpr const char* MAPPER_NAME = "mapper"; static constexpr const char* APP_BASE_HEADER = "flex/engines/apps/cypher_app_base.h"; -static constexpr const char* APP_BASE_CLASS_NAME = "HqpsAppBase"; +static constexpr const char* APP_BASE_CLASS_NAME = "AppBase"; static constexpr const char* QUERY_FUNC_RETURN = "results::CollectiveResults"; enum class StorageBackend { diff --git a/flex/codegen/src/hqps_generator.h b/flex/codegen/src/hqps_generator.h index cf95ce15823b..5f096fe0c5ba 100644 --- a/flex/codegen/src/hqps_generator.h +++ b/flex/codegen/src/hqps_generator.h @@ -43,7 +43,7 @@ static constexpr const char* QUERY_TEMPLATE_STR = "// DO NOT EDIT\n" "\n" "#include \"flex/engines/hqps_db/core/sync_engine.h\"\n" - "#include \"flex/engines/hqps_db/app/hqps_app_base.h\"\n" // app_base_header.h + "#include \"flex/engines/graph_db/app/app_base.h\"\n" // app_base_header.h "#include \"%1%\"\n" // graph_interface_header.h "\n" "\n" @@ -52,38 +52,43 @@ static constexpr const char* QUERY_TEMPLATE_STR = "%2%\n" "\n" "// Auto generated query class definition\n" - "class %3% : public HqpsAppBase<%4%> {\n" + "class %3% : public AppBase {\n" " public:\n" " using Engine = SyncEngine<%4%>;\n" " using label_id_t = typename %4%::label_id_t;\n" " using vertex_id_t = typename %4%::vertex_id_t;\n" + " // constructor\n" + " %3%(const GraphDBSession& session) : %6%(session) {}\n" "// Query function for query class\n" - " %5% Query(const %4%& %6% %7%) const{\n" + " %5% Query(%7%) const{\n" " %8%\n" " }\n" "// Wrapper query function for query class\n" - " %5% Query(const %4%& %6%, Decoder& decoder) const override {\n" + " bool Query(Decoder& decoder, Encoder& encoder) override {\n" " //decoding params from decoder, and call real query func\n" " %9%\n" - " return Query(%6% %10%);\n" + " auto res = Query(%10%);\n" + " // dump results to string\n" + " std::string res_str = res.SerializeAsString();\n" + " // encode results to encoder\n" + " encoder.put_string(res_str);\n" + " return true;\n" " }\n" + " //private members\n" + " private:\n" + " %4% %6%;\n" "};\n" "} // namespace gs\n" "\n" "// extern c interfaces\n" "extern \"C\" {\n" - "void* CreateApp(gs::GraphStoreType store_type) {\n" - " if (store_type == %11%) {\n" - " gs::%3%* app =\n" - " new gs::%3%();\n" - " return static_cast(app);\n" - " }\n" - " return nullptr;\n" + "void* CreateApp(gs::GraphDBSession& db) {\n" + " gs::%3%* app = new gs::%3%(db);\n" + " return static_cast(app);\n" "}\n" - "void DeleteApp(void* app, gs::GraphStoreType store_type) {\n" - " if (store_type == %11%) {\n" - " gs::%3%* casted =\n" - " static_cast(app);\n" + "void DeleteApp(void* app) {\n" + " if (app != nullptr) {\n" + " gs::%3%* casted = static_cast(app);\n" " delete casted;\n" " }\n" "}\n" @@ -195,7 +200,7 @@ class QueryGenerator { formater % ctx_.GetGraphHeader() % expr_code % ctx_.GetQueryClassName() % ctx_.GetGraphInterface() % ctx_.GetQueryRet() % ctx_.GraphVar() % dynamic_vars_str % query_code % decoding_params_code % - decoded_params_str % storage_backend_to_string(ctx_.GetStorageType()); + decoded_params_str; return formater.str(); } diff --git a/flex/engines/graph_db/CMakeLists.txt b/flex/engines/graph_db/CMakeLists.txt index 288bb0e15487..9acdee1d1465 100644 --- a/flex/engines/graph_db/CMakeLists.txt +++ b/flex/engines/graph_db/CMakeLists.txt @@ -3,7 +3,8 @@ file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc" add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES}) -target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(flex_graph_db PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils hqps_plan_proto ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) install(TARGETS flex_graph_db RUNTIME DESTINATION bin diff --git a/flex/engines/graph_db/app/app_base.cc b/flex/engines/graph_db/app/app_base.cc index b66368779cec..4eac06d1b5fe 100644 --- a/flex/engines/graph_db/app/app_base.cc +++ b/flex/engines/graph_db/app/app_base.cc @@ -23,9 +23,24 @@ namespace gs { SharedLibraryAppFactory::SharedLibraryAppFactory(const std::string& path) : app_path_(path) { app_handle_ = dlopen(app_path_.c_str(), RTLD_LAZY); + auto* p_error_msg = dlerror(); + if (p_error_msg) { + LOG(ERROR) << "Fail to open library: " << path + << ", error: " << p_error_msg; + } *(void**) (&func_creator_) = dlsym(app_handle_, "CreateApp"); + p_error_msg = dlerror(); + if (p_error_msg) { + LOG(ERROR) << "Failed to get symbol CreateApp from " << path + << ". Reason: " << std::string(p_error_msg); + } *(void**) (&func_deletor_) = dlsym(app_handle_, "DeleteApp"); + p_error_msg = dlerror(); + if (p_error_msg) { + LOG(ERROR) << "Failed to get symbol DeleteApp from " << path + << ". Reason: " << std::string(p_error_msg); + } } SharedLibraryAppFactory::~SharedLibraryAppFactory() { @@ -35,6 +50,11 @@ SharedLibraryAppFactory::~SharedLibraryAppFactory() { } AppWrapper SharedLibraryAppFactory::CreateApp(GraphDBSession& db) { + if (func_creator_ == NULL) { + LOG(ERROR) << "Failed to create app from " << app_path_ + << ". Reason: func_creator_ is NULL"; + return AppWrapper(); + } AppBase* app = static_cast(func_creator_(db)); return AppWrapper(app, func_deletor_); } diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index 0d0ce3681538..cb6de7600676 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -18,6 +18,7 @@ #include "flex/engines/graph_db/app/server_app.h" #include "flex/engines/graph_db/database/wal.h" +#include "flex/utils/yaml_utils.h" namespace gs { @@ -48,6 +49,64 @@ GraphDB& GraphDB::get() { return db; } +Result GraphDB::Open(const Schema& schema, const std::string& data_dir, + int thread_num) { + std::filesystem::path data_dir_path(data_dir); + std::filesystem::path serial_path = data_dir_path / "init_snapshot.bin"; + if (!std::filesystem::exists(data_dir_path)) { + return Result(StatusCode::NotExists, "Data directory does not exist", + false); + } + if (!std::filesystem::exists(serial_path)) { + return Result(StatusCode::NotExists, "Snapshot file does not exist", + false); + } + LOG(INFO) << "Initializing graph db from data files of work directory"; + + // Now assign the new thread_num + thread_num_ = thread_num; + try { + graph_.Deserialize(data_dir_path.string()); + } catch (std::exception& e) { + LOG(ERROR) << "Exception: " << e.what(); + return Result(StatusCode::InternalError, + "Exception: " + std::string(e.what()), false); + } + VLOG(10) << "Finish deserializing graph db"; + if (!graph_.schema().Equals(schema)) { + return Result(StatusCode::InternalError, + "Schema of work directory is not compatible with the " + "graph schema", + false); + } + // Set the plugin info from schema to graph_.schema(), since the plugin info + // is not serialized and deserialized. + auto& mutable_schema = graph_.mutable_schema(); + mutable_schema.SetPluginDir(schema.GetPluginDir()); + std::vector plugin_paths; + for (auto plugin_pair : schema.GetPlugins()) { + plugin_paths.emplace_back(plugin_pair.first); + } + mutable_schema.EmplacePlugins(plugin_paths); + + openWalAndCreateContexts(data_dir_path); + return Result(true); +} + +void GraphDB::Close() { + //-----------Clear graph_db---------------- + graph_.Clear(); + version_manager_.clear(); + if (contexts_ != nullptr) { + for (int i = 0; i < thread_num_; ++i) { + contexts_[i].~SessionLocalContext(); + } + free(contexts_); + } + std::fill(app_paths_.begin(), app_paths_.end(), ""); + std::fill(app_factories_.begin(), app_factories_.end(), nullptr); +} + void GraphDB::Init(const Schema& schema, const LoadingConfig& load_config, const std::string& data_dir, int thread_num) { std::filesystem::path data_dir_path(data_dir); @@ -84,33 +143,12 @@ void GraphDB::Init(const Schema& schema, const LoadingConfig& load_config, } graph_.Deserialize(data_dir_path.string()); if (!graph_.schema().Equals(schema)) { - LOG(FATAL) - << "Schema of work directory is not compatible with the given schema"; + LOG(FATAL) << "Schema of work directory is not compatible with the " + "given schema"; } } - - std::filesystem::path wal_dir = data_dir_path / "wal"; - if (!std::filesystem::exists(wal_dir)) { - std::filesystem::create_directory(wal_dir); - } - std::vector wal_files; - for (const auto& entry : std::filesystem::directory_iterator(wal_dir)) { - wal_files.push_back(entry.path().string()); - } - thread_num_ = thread_num; - contexts_ = static_cast( - aligned_alloc(4096, sizeof(SessionLocalContext) * thread_num)); - for (int i = 0; i < thread_num_; ++i) { - new (&contexts_[i]) SessionLocalContext(*this, i); - } - ingestWals(wal_files, thread_num_); - - for (int i = 0; i < thread_num_; ++i) { - contexts_[i].logger.open(wal_dir.string(), i); - } - - initApps(schema.GetPluginsList()); + openWalAndCreateContexts(data_dir); } ReadTransaction GraphDB::GetReadTransaction() { @@ -162,27 +200,28 @@ AppWrapper GraphDB::CreateApp(uint8_t app_type, int thread_id) { } } -void GraphDB::registerApp(const std::string& path, uint8_t index) { +bool GraphDB::registerApp(const std::string& plugin_path, uint8_t index) { // this function will only be called when initializing the graph db - if (index == 0) { - for (size_t i = 1; i != 256; ++i) { - if (app_factories_[i] == nullptr) { - index = static_cast(i); - break; - } - } - } - if (index == 0) { - LOG(ERROR) << "too many stored procedures..."; + VLOG(10) << "Registering stored procedure at:" << std::to_string(index) + << ", path:" << plugin_path; + if (!app_factories_[index] && app_paths_[index].empty()) { + app_paths_[index] = plugin_path; + app_factories_[index] = + std::make_shared(plugin_path); + return true; + } else { + LOG(ERROR) << "Stored procedure has already been registered at:" + << std::to_string(index) << ", path:" << app_paths_[index]; + return false; } - app_paths_[index] = path; - app_factories_[index] = std::make_shared(path); } void GraphDB::GetAppInfo(Encoder& output) { std::string ret; for (size_t i = 1; i != 256; ++i) { - output.put_string(app_paths_[i]); + if (!app_paths_.empty()) { + output.put_string(app_paths_[i]); + } } } @@ -235,16 +274,52 @@ void GraphDB::ingestWals(const std::vector& wals, int thread_num) { version_manager_.init_ts(parser.last_ts()); } -void GraphDB::initApps(const std::vector& plugins) { +void GraphDB::initApps( + const std::unordered_map>& + plugins) { + VLOG(1) << "Initializing stored procedures, size: " << plugins.size() + << " ..."; for (size_t i = 0; i < 256; ++i) { app_factories_[i] = nullptr; } app_factories_[0] = std::make_shared(); + size_t valid_plugins = 0; + for (auto& path_and_index : plugins) { + auto path = path_and_index.second.first; + auto index = path_and_index.second.second; + if (registerApp(path, index)) { + ++valid_plugins; + } + } + LOG(INFO) << "Successfully registered stored procedures : " << valid_plugins + << ", from " << plugins.size(); +} - uint8_t sp_index = 1; - for (auto path : plugins) { - registerApp(path, sp_index++); +void GraphDB::openWalAndCreateContexts( + const std::filesystem::path& data_dir_path) { + std::filesystem::path wal_dir = data_dir_path / "wal"; + if (!std::filesystem::exists(wal_dir)) { + std::filesystem::create_directory(wal_dir); + } + std::vector wal_files; + for (const auto& entry : std::filesystem::directory_iterator(wal_dir)) { + wal_files.push_back(entry.path().string()); + } + + contexts_ = static_cast( + aligned_alloc(4096, sizeof(SessionLocalContext) * thread_num_)); + for (int i = 0; i < thread_num_; ++i) { + new (&contexts_[i]) SessionLocalContext(*this, i); + } + ingestWals(wal_files, thread_num_); + + for (int i = 0; i < thread_num_; ++i) { + contexts_[i].logger.open(wal_dir.string(), i); } + VLOG(1) << "Successfully restore graph db from data directory"; + + initApps(graph_.schema().GetPlugins()); + VLOG(1) << "Successfully restore load plugins"; } } // namespace gs diff --git a/flex/engines/graph_db/database/graph_db.h b/flex/engines/graph_db/database/graph_db.h index a85bae4d99e3..d935329e72eb 100644 --- a/flex/engines/graph_db/database/graph_db.h +++ b/flex/engines/graph_db/database/graph_db.h @@ -50,6 +50,21 @@ class GraphDB { void Init(const Schema& schema, const LoadingConfig& config, const std::string& data_dir, int thread_num = 1); + /** + * @brief Load the graph from data directory. + * @param schema The schema of graph. It should be the same as the schema, + * except that the procedure enable_lists changes. + * @param data_dir The directory of graph data. + * @param thread_num The number of threads for graph db concurrency + */ + Result Open(const Schema& schema, const std::string& data_dir, + int32_t thread_num); + + /** + * @brief Close the current opened graph. + */ + void Close(); + /** @brief Create a transaction to read vertices and edges. * * @return graph_dir The directory of graph data. @@ -102,11 +117,15 @@ class GraphDB { int SessionNum() const; private: - void registerApp(const std::string& path, uint8_t index = 0); + bool registerApp(const std::string& path, uint8_t index = 0); void ingestWals(const std::vector& wals, int thread_num); - void initApps(const std::vector& plugins); + void initApps( + const std::unordered_map>& + plugins); + + void openWalAndCreateContexts(const std::filesystem::path& data_dir_path); friend class GraphDBSession; diff --git a/flex/engines/graph_db/database/graph_db_session.cc b/flex/engines/graph_db/database/graph_db_session.cc index dd22f692315d..fbbc54884e57 100644 --- a/flex/engines/graph_db/database/graph_db_session.cc +++ b/flex/engines/graph_db/database/graph_db_session.cc @@ -20,6 +20,27 @@ namespace gs { +void put_argment(Encoder& encoder, const query::Argument& argment) { + auto& value = argment.value(); + auto item_case = value.item_case(); + switch (item_case) { + case common::Value::kI32: + encoder.put_int(value.i32()); + break; + case common::Value::kI64: + encoder.put_long(value.i64()); + break; + case common::Value::kF64: + encoder.put_double(value.f64()); + break; + case common::Value::kStr: + encoder.put_string(value.str()); + break; + default: + LOG(ERROR) << "Not recognizable param type" << static_cast(item_case); + } +} + ReadTransaction GraphDBSession::GetReadTransaction() { uint32_t ts = db_.version_manager_.acquire_read_timestamp(); return ReadTransaction(db_.graph_, db_.version_manager_, ts); @@ -96,7 +117,7 @@ std::shared_ptr GraphDBSession::get_vertex_id_column( #define likely(x) __builtin_expect(!!(x), 1) -std::vector GraphDBSession::Eval(const std::string& input) { +Result> GraphDBSession::Eval(const std::string& input) { uint8_t type = input.back(); const char* str_data = input.data(); size_t str_len = input.size() - 1; @@ -114,51 +135,152 @@ std::vector GraphDBSession::Eval(const std::string& input) { if (app_wrappers_[type].app() == NULL) { LOG(ERROR) << "[Query-" + std::to_string((int) type) << "] is not registered..."; - return result_buffer; + return Result>( + StatusCode::NotExists, + "Query:" + std::to_string((int) type) + " is not registere", + result_buffer); } else { apps_[type] = app_wrappers_[type].app(); app = apps_[type]; } } - if (app->Query(decoder, encoder)) { - return result_buffer; + for (auto i = 0; i < MAX_RETRY; ++i) { + if (app->Query(decoder, encoder)) { + return result_buffer; + } + + LOG(INFO) << "[Query-" << (int) type << "][Thread-" << thread_id_ + << "] retry - " << i << " / " << MAX_RETRY; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + decoder.reset(str_data, str_len); + result_buffer.clear(); } - LOG(INFO) << "[Query-" << (int) type << "][Thread-" << thread_id_ - << "] retry - 1 / 3"; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return Result>( + StatusCode::QueryFailed, + "Query failed for procedure id:" + std::to_string((int) type), + result_buffer); +} + +// Evaluating stored procedure for hqps adhoc query, the dynamic lib is closed +// immediately after the query +Result> GraphDBSession::EvalAdhoc( + const std::string& input_lib_path) { + std::vector result_buffer; + std::vector input_buffer; // empty. Adhoc query receives no input + Decoder decoder(input_buffer.data(), input_buffer.size()); + Encoder encoder(result_buffer); + + // the dynamic library will automatically be closed after the query + auto app_factory = std::make_shared(input_lib_path); + AppWrapper app_wrapper; // wrapper should be destroyed before the factory - decoder.reset(str_data, str_len); - result_buffer.clear(); - if (app->Query(decoder, encoder)) { - return result_buffer; + if (app_factory) { + app_wrapper = app_factory->CreateApp(*this); + if (app_wrapper.app() == NULL) { + LOG(ERROR) << "Fail to create app for adhoc query: " << input_lib_path; + return Result>( + StatusCode::InternalError, + "Fail to create app for: " + input_lib_path, result_buffer); + } + } else { + LOG(ERROR) << "Fail to evaluate adhoc query: " << input_lib_path; + return Result>( + StatusCode::NotExists, + "Fail to open dynamic lib for: " + input_lib_path, result_buffer); } - LOG(INFO) << "[Query-" << (int) type << "][Thread-" << thread_id_ - << "] retry - 2 / 3"; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + for (auto i = 0; i < MAX_RETRY; ++i) { + if (app_wrapper.app()->Query(decoder, encoder)) { + return result_buffer; + } - decoder.reset(str_data, str_len); - result_buffer.clear(); - if (app->Query(decoder, encoder)) { - return result_buffer; + LOG(INFO) << "[Query-" << input_lib_path << "][Thread-" << thread_id_ + << "] retry - " << i << " / " << MAX_RETRY; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + result_buffer.clear(); } + return Result>( + StatusCode::QueryFailed, + "Query failed for adhoc query: " + input_lib_path, result_buffer); +} - LOG(INFO) << "[Query-" << (int) type << "][Thread-" << thread_id_ - << "] retry - 3 / 3"; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); +Result> GraphDBSession::EvalHqpsProcedure( + const query::Query& query_pb) { + auto query_name = query_pb.query_name().name(); + if (query_name.empty()) { + LOG(ERROR) << "Query name is empty"; + return Result>(StatusCode::InValidArgument, + "Query name is empty", {}); + } + auto& app_name_to_path_index = db_.schema().GetPlugins(); + // get procedure id from name. + if (app_name_to_path_index.count(query_name) <= 0) { + LOG(ERROR) << "Query name is not registered: " << query_name; + return Result>( + StatusCode::NotExists, "Query name is not registered: " + query_name, + {}); + } - decoder.reset(str_data, str_len); - result_buffer.clear(); - if (app->Query(decoder, encoder)) { - return result_buffer; + // get app + auto type = app_name_to_path_index.at(query_name).second; + if (type >= apps_.size()) { + LOG(ERROR) << "Query type is not registered: " << type; + return Result>( + StatusCode::NotExists, + "Query type is not registered: " + std::to_string(type), {}); + } + AppBase* app = nullptr; + if (likely(apps_[type] != nullptr)) { + app = apps_[type]; + } else { + app_wrappers_[type] = db_.CreateApp(type, thread_id_); + if (app_wrappers_[type].app() == NULL) { + LOG(ERROR) << "[Query-" + std::to_string((int) type) + << "] is not registered..."; + return Result>( + StatusCode::NotExists, + "Query:" + std::to_string((int) type) + " is not registered", {}); + } else { + apps_[type] = app_wrappers_[type].app(); + app = apps_[type]; + } + } + + if (app == nullptr) { + LOG(ERROR) << "Query type is not registered: " << type + << ", query name: " << query_name; + return Result>( + StatusCode::NotExists, + "Query type is not registered: " + std::to_string(type), {}); } - LOG(INFO) << "[Query-" << (int) type << "][Thread-" << thread_id_ - << "] failed after 3 retries"; - result_buffer.clear(); - return result_buffer; + std::vector input_buffer; + gs::Encoder input_encoder(input_buffer); + auto& args = query_pb.arguments(); + for (auto i = 0; i < args.size(); ++i) { + auto& arg = args[i]; + put_argment(input_encoder, arg); + } + const char* str_data = input_buffer.data(); + size_t str_len = input_buffer.size(); + gs::Decoder input_decoder(input_buffer.data(), input_buffer.size()); + + for (auto i = 0; i < MAX_RETRY; ++i) { + std::vector result_buffer; + gs::Encoder result_encoder(result_buffer); + if (app->Query(input_decoder, result_encoder)) { + return Result>(std::move(result_buffer)); + } + LOG(INFO) << "[Query-" << query_name << "][Thread-" << thread_id_ + << "] retry - " << i << " / " << MAX_RETRY; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + input_decoder.reset(str_data, str_len); + } + return Result>( + StatusCode::QueryFailed, "Query failed for procedure: " + query_name, {}); } #undef likely diff --git a/flex/engines/graph_db/database/graph_db_session.h b/flex/engines/graph_db/database/graph_db_session.h index 4c12c4696a00..c1fc48240b31 100644 --- a/flex/engines/graph_db/database/graph_db_session.h +++ b/flex/engines/graph_db/database/graph_db_session.h @@ -22,8 +22,10 @@ #include "flex/engines/graph_db/database/single_edge_insert_transaction.h" #include "flex/engines/graph_db/database/single_vertex_insert_transaction.h" #include "flex/engines/graph_db/database/update_transaction.h" +#include "flex/proto_generated_gie/stored_procedure.pb.h" #include "flex/storages/rt_mutable_graph/mutable_property_fragment.h" #include "flex/utils/property/column.h" +#include "flex/utils/result.h" namespace gs { @@ -31,8 +33,11 @@ class GraphDB; class WalWriter; class ArenaAllocator; +void put_argment(gs::Encoder& encoder, const query::Argument& argment); + class GraphDBSession { public: + static constexpr int32_t MAX_RETRY = 4; GraphDBSession(GraphDB& db, ArenaAllocator& alloc, WalWriter& logger, int thread_id) : db_(db), alloc_(alloc), logger_(logger), thread_id_(thread_id) { @@ -63,7 +68,14 @@ class GraphDBSession { // Get vertex id column. std::shared_ptr get_vertex_id_column(uint8_t label) const; - std::vector Eval(const std::string& input); + Result> Eval(const std::string& input); + + // Evaluate a temporary stored procedure. close the handle of the dynamic lib + // immediately. + Result> EvalAdhoc(const std::string& input_lib_path); + + // Evaluate a stored procedure with input parameters given. + Result> EvalHqpsProcedure(const query::Query& query_pb); void GetAppInfo(Encoder& result); diff --git a/flex/engines/graph_db/database/version_manager.cc b/flex/engines/graph_db/database/version_manager.cc index df78f997226e..ea09e2979441 100644 --- a/flex/engines/graph_db/database/version_manager.cc +++ b/flex/engines/graph_db/database/version_manager.cc @@ -33,6 +33,13 @@ void VersionManager::init_ts(uint32_t ts) { read_ts_.store(ts); } +void VersionManager::clear() { + write_ts_.store(1); + read_ts_.store(0); + pending_reqs_.store(0); + buf_.clear(); +} + uint32_t VersionManager::acquire_read_timestamp() { auto pr = pending_reqs_.fetch_add(1); if (likely(pr >= 0)) { diff --git a/flex/engines/graph_db/database/version_manager.h b/flex/engines/graph_db/database/version_manager.h index 6a14f68d9a3e..2422cccd37b6 100644 --- a/flex/engines/graph_db/database/version_manager.h +++ b/flex/engines/graph_db/database/version_manager.h @@ -38,6 +38,8 @@ class VersionManager { void init_ts(uint32_t ts); + void clear(); + uint32_t acquire_read_timestamp(); void release_read_timestamp(); diff --git a/flex/engines/graph_db/grin/src/topology/structure.cc b/flex/engines/graph_db/grin/src/topology/structure.cc index 73b3d053669a..a3dcdac18977 100644 --- a/flex/engines/graph_db/grin/src/topology/structure.cc +++ b/flex/engines/graph_db/grin/src/topology/structure.cc @@ -68,7 +68,7 @@ GRIN_GRAPH grin_get_graph_from_storage(const char* uri) { } auto schema = gs::Schema::LoadFromYaml(graph_schema_path); auto loading_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_config_path); + gs::LoadingConfig::ParseFromYamlFile(schema, bulk_load_config_path); GRIN_GRAPH_T* g = new GRIN_GRAPH_T(); auto loader = diff --git a/flex/engines/hqps_db/app/hqps_app_base.h b/flex/engines/hqps_db/app/hqps_app_base.h deleted file mode 100644 index 504140e26c16..000000000000 --- a/flex/engines/hqps_db/app/hqps_app_base.h +++ /dev/null @@ -1,48 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef ENGINES_HQPS_APP_CYPHER_APP_BASE_H_ -#define ENGINES_HQPS_APP_CYPHER_APP_BASE_H_ - -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" -#include "flex/proto_generated_gie/results.pb.h" -#include "flex/utils/app_utils.h" - -namespace gs { - -enum class GraphStoreType { - Grape = 0, -}; - -template -class HqpsAppBase { - public: - /** - * @brief Construct a new Hqps App Base object - */ - virtual ~HqpsAppBase() = default; - /** - * @brief Query the graph with the given input - * - * @param graph The graph to query - * @param input The input to query - * @return virtual results::CollectiveResults The query result - */ - virtual results::CollectiveResults Query(const GRAPH_TYPE& graph, - Decoder& input) const = 0; -}; - -} // namespace gs - -#endif // ENGINES_HQPS_APP_CYPHER_APP_BASE_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/core/utils/hqps_utils.h b/flex/engines/hqps_db/core/utils/hqps_utils.h index a7f24ae210e2..49b77202cbdc 100644 --- a/flex/engines/hqps_db/core/utils/hqps_utils.h +++ b/flex/engines/hqps_db/core/utils/hqps_utils.h @@ -988,18 +988,6 @@ struct QPSError { std::string GetMessage() { return message; } }; -class QPSException : public std::exception { - public: - explicit QPSException(std::string&& error_msg) - : std::exception(), _err_msg(error_msg) {} - ~QPSException() override = default; - - const char* what() const noexcept override { return _err_msg.c_str(); } - - private: - std::string _err_msg; -}; - template struct function_traits : public function_traits {}; // For generic types, directly use the result of the signature of its diff --git a/flex/engines/http_server/CMakeLists.txt b/flex/engines/http_server/CMakeLists.txt index 0ddd2a13bff4..f26a9217b581 100644 --- a/flex/engines/http_server/CMakeLists.txt +++ b/flex/engines/http_server/CMakeLists.txt @@ -13,7 +13,7 @@ if (Hiactor_FOUND) target_compile_options (flex_server PUBLIC -Wno-attributes) - target_link_libraries(flex_server Hiactor::hiactor hqps_plan_proto) + target_link_libraries(flex_server Hiactor::hiactor hqps_plan_proto flex_graph_db) target_include_directories(flex_server PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/../hqps/) diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc new file mode 100644 index 000000000000..4a91f0a5c9cb --- /dev/null +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -0,0 +1,387 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/http_server/actor/admin_actor.act.h" + +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/engines/http_server/codegen_proxy.h" +#include "flex/engines/http_server/workdir_manipulator.h" +#include "flex/utils/service_utils.h" +#include "nlohmann/json.hpp" + +#include + +namespace server { + +admin_actor::~admin_actor() { + // finalization + // ... +} + +admin_actor::admin_actor(hiactor::actor_base* exec_ctx, + const hiactor::byte_t* addr) + : hiactor::actor(exec_ctx, addr) { + set_max_concurrency(1); // set max concurrency for task reentrancy (stateful) + // initialization + // ... +} + +// Create a new Graph with the passed graph config. +seastar::future admin_actor::run_create_graph( + query_param&& query_param) { + LOG(INFO) << "Creating Graph: " << query_param.content; + + YAML::Node yaml; + + try { + nlohmann::json json = nlohmann::json::parse(query_param.content); + std::stringstream json_ss; + json_ss << query_param.content; + yaml = YAML::Load(json_ss); + } catch (std::exception& e) { + LOG(ERROR) << "Fail to parse json: " << e.what(); + return seastar::make_exception_future( + std::runtime_error("Fail to parse json: " + std::string(e.what()))); + } catch (...) { + LOG(ERROR) << "Fail to parse json: " << query_param.content; + return seastar::make_exception_future( + std::runtime_error("Fail to parse json: " + query_param.content)); + } + + auto result = server::WorkDirManipulator::CreateGraph(yaml); + + if (result.ok()) { + VLOG(10) << "Successfully created graph"; + return seastar::make_ready_future(std::move(result.value())); + } else { + LOG(ERROR) << "Fail to create graph: " << result.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to create graph: " + result.status().error_message())); + } +} + +// get graph schema +// query_param is the graph name +seastar::future admin_actor::run_get_graph_schema( + query_param&& query_param) { + LOG(INFO) << "Get Graph schema for graph: " << query_param.content; + + auto schema_result = + server::WorkDirManipulator::GetGraphSchemaString(query_param.content); + if (schema_result.ok()) { + return seastar::make_ready_future( + std::move(schema_result.value())); + } else { + LOG(ERROR) << "Fail to get graph schema: " + << schema_result.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to get graph schema: " + schema_result.status().error_message())); + } +} + +// list all graphs +seastar::future admin_actor::run_list_graphs( + query_param&& query_param) { + LOG(INFO) << "List all graphs."; + auto list_result = server::WorkDirManipulator::ListGraphs(); + if (!list_result.ok()) { + LOG(ERROR) << "Fail to list graphs: " + << list_result.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to list graphs: " + list_result.status().error_message())); + } else { + VLOG(10) << "Successfully list graphs"; + return seastar::make_ready_future( + std::move(list_result.value())); + } +} + +// delete one graph +seastar::future admin_actor::run_delete_graph( + query_param&& query_param) { + LOG(INFO) << "Delete graph: " << query_param.content; + + auto delete_res = + server::WorkDirManipulator::DeleteGraph(query_param.content); + if (delete_res.ok()) { + return seastar::make_ready_future( + std::move(delete_res.value())); + } else { + LOG(ERROR) << "Fail to delete graph: " + << delete_res.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to delete graph: " + delete_res.status().error_message())); + } +} + +// load the graph. +seastar::future admin_actor::run_graph_loading( + graph_management_param&& query_param) { + // query_param constains two parameter, first for graph name, second for graph + // config + auto content = query_param.content; + auto& graph_name = content.first; + VLOG(1) << "Parse json payload for graph: " << graph_name; + auto& graph_config = content.second; + + YAML::Node yaml; + try { + // parse json from query_param.content + nlohmann::json json = nlohmann::json::parse(graph_config); + std::stringstream json_ss; + json_ss << graph_config; + yaml = YAML::Load(json_ss); + } catch (std::exception& e) { + LOG(ERROR) << "Fail to parse json: " << e.what(); + return seastar::make_exception_future( + std::runtime_error("Fail to parse json: " + std::string(e.what()))); + } catch (...) { + LOG(ERROR) << "Fail to parse json: " << graph_config; + return seastar::make_exception_future(std::runtime_error( + "Fail to parse json when running dataloading for : " + graph_name)); + } + int32_t loading_thread_num = 1; + if (yaml["loading_thread_num"]) { + loading_thread_num = yaml["loading_thread_num"].as(); + } + + auto graph_loading_res = server::WorkDirManipulator::LoadGraph( + graph_name, yaml, loading_thread_num); + + if (graph_loading_res.ok()) { + VLOG(10) << "Successfully loaded graph"; + return seastar::make_ready_future( + std::move(graph_loading_res.value())); + } else { + LOG(ERROR) << "Fail to load graph: " + << graph_loading_res.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to load graph: " + graph_loading_res.status().error_message())); + } +} + +// Get all procedure with graph_name and procedure_name +seastar::future admin_actor::get_procedure_by_procedure_name( + procedure_query_param&& query_param) { + auto& graph_name = query_param.content.first; + auto& procedure_name = query_param.content.second; + LOG(INFO) << "Get procedure: " << procedure_name + << " for graph: " << graph_name; + auto get_procedure_res = + server::WorkDirManipulator::GetProcedureByGraphAndProcedureName( + graph_name, procedure_name); + if (get_procedure_res.ok()) { + VLOG(10) << "Successfully get procedure procedures"; + return seastar::make_ready_future( + std::move(get_procedure_res.value())); + } else { + LOG(ERROR) << "Fail to get procedure for graph: " << graph_name + << " and procedure: " << procedure_name << ", error message: " + << get_procedure_res.status().error_message(); + return seastar::make_exception_future( + std::runtime_error("Fail to get procedure: " + + get_procedure_res.status().error_message())); + } +} + +// Get all procedures of one graph. +seastar::future admin_actor::get_procedures_by_graph_name( + query_param&& query_param) { + auto& graph_name = query_param.content; + auto get_all_procedure_res = + server::WorkDirManipulator::GetProceduresByGraphName(graph_name); + if (get_all_procedure_res.ok()) { + VLOG(10) << "Successfully get all procedures: " + << get_all_procedure_res.value(); + return seastar::make_ready_future( + std::move(get_all_procedure_res.value())); + } else { + LOG(ERROR) << "Fail to get all procedures: " + << get_all_procedure_res.status().error_message(); + return seastar::make_exception_future( + std::runtime_error("Fail to get all procedures: " + + get_all_procedure_res.status().error_message())); + } +} + +seastar::future admin_actor::create_procedure( + create_procedure_query_param&& query_param) { + auto& graph_name = query_param.content.first; + auto& parameter = query_param.content.second; + return server::WorkDirManipulator::CreateProcedure(graph_name, parameter) + .then_wrapped([](auto&& f) { + try { + auto res = f.get(); + return seastar::make_ready_future( + query_result{std::move(res)}); + } catch (std::exception& e) { + LOG(ERROR) << "Fail to create procedure: " << e.what(); + return seastar::make_exception_future( + std::runtime_error("Fail to create procedure: " + + std::string(e.what()))); + } + }); +} + +// Delete a procedure by graph name and procedure name +seastar::future admin_actor::delete_procedure( + create_procedure_query_param&& query_param) { + auto& graph_name = query_param.content.first; + auto& procedure_name = query_param.content.second; + auto delete_procedure_res = + server::WorkDirManipulator::DeleteProcedure(graph_name, procedure_name); + if (delete_procedure_res.ok()) { + VLOG(10) << "Successfully get all procedures"; + return seastar::make_ready_future( + std::move(delete_procedure_res.value())); + } else { + LOG(ERROR) << "Fail to create procedure: " + << delete_procedure_res.status().error_message(); + return seastar::make_exception_future( + std::runtime_error("Fail to create procedures: " + + delete_procedure_res.status().error_message())); + } +} + +// update a procedure by graph name and procedure name +seastar::future admin_actor::update_procedure( + update_procedure_query_param&& query_param) { + auto& graph_name = std::get<0>(query_param.content); + auto& procedure_name = std::get<1>(query_param.content); + auto& parameter = std::get<2>(query_param.content); + auto update_procedure_res = server::WorkDirManipulator::UpdateProcedure( + graph_name, procedure_name, parameter); + if (update_procedure_res.ok()) { + VLOG(10) << "Successfully update procedure: " << procedure_name; + return seastar::make_ready_future( + std::move(update_procedure_res.value())); + } else { + LOG(ERROR) << "Fail to create procedure: " + << update_procedure_res.status().error_message(); + return seastar::make_exception_future( + std::runtime_error("Fail to create procedures: " + + update_procedure_res.status().error_message())); + } +} + +// Manage the service of one graph. +seastar::future admin_actor::start_service( + query_param&& query_param) { + // parse query_param.content as json and get graph_name + auto& content = query_param.content; + std::string graph_name; + try { + if (!content.empty()) { + nlohmann::json json = nlohmann::json::parse(content); + if (json.contains("graph_name")) { + graph_name = json["graph_name"].get(); + } + } else { + graph_name = server::WorkDirManipulator::GetRunningGraph(); + LOG(WARNING) + << "Request payload is empty, will restart on current graph: " + << server::WorkDirManipulator::GetRunningGraph(); + } + LOG(WARNING) << "Starting service with graph: " << graph_name; + + auto schema_result = server::WorkDirManipulator::GetGraphSchema(graph_name); + if (!schema_result.ok()) { + LOG(ERROR) << "Fail to get graph schema: " + << schema_result.status().error_message() << ", " + << graph_name; + return seastar::make_exception_future(std::runtime_error( + "Fail to get graph schema: " + + schema_result.status().error_message() + ", " + graph_name)); + } + auto data_dir = server::WorkDirManipulator::GetDataDirectory(graph_name); + if (!data_dir.ok()) { + LOG(ERROR) << "Fail to get data directory: " + << data_dir.status().error_message(); + return seastar::make_exception_future(std::runtime_error( + "Fail to get data directory: " + data_dir.status().error_message())); + } + { + std::lock_guard lock(mtx_); + auto& db = gs::GraphDB::get(); + LOG(INFO) << "Update service running on graph:" << graph_name; + auto& schema_value = schema_result.value(); + // use the previous thread num + auto thread_num = db.SessionNum(); + db.Close(); + if (!db.Open(schema_value, data_dir.value(), thread_num).ok()) { + LOG(ERROR) << "Fail to load graph from data directory: " + << data_dir.value(); + return seastar::make_exception_future(std::runtime_error( + "Fail to load graph from data directory: " + data_dir.value())); + } + server::WorkDirManipulator::SetRunningGraph(graph_name); + } + + LOG(INFO) << "Successfully started service with graph: " << graph_name; + + return seastar::make_ready_future( + "Successfully start service"); + } catch (std::exception& e) { + LOG(ERROR) << "Fail to Start service: "; + return seastar::make_exception_future( + std::runtime_error(e.what())); + } +} + +// get service status +seastar::future admin_actor::service_status( + query_param&& query_param) { + auto& hqps_service = HQPSService::get(); + auto query_port = hqps_service.get_query_port(); + nlohmann::json res; + if (query_port != 0) { + res["status"] = "running"; + res["query_port"] = query_port; + res["graph_name"] = server::WorkDirManipulator::GetRunningGraph(); + } else { + LOG(INFO) << "Query service has not been inited!"; + res["status"] = "Query service has not been inited!"; + } + return seastar::make_ready_future(std::move(res.dump())); +} + +// get node status. +seastar::future admin_actor::node_status( + query_param&& query_param) { + // get current host' cpu usage and memory usage + auto cpu_usage = gs::get_current_cpu_usage(); + auto mem_usage = gs::get_total_physical_memory_usage(); + // construct the result json string + nlohmann::json json; + { + std::stringstream ss; + if (cpu_usage.first < 0 || cpu_usage.second <= 0) { + ss << "cpu_usage is not available"; + } else { + ss << "cpu_usage is " << cpu_usage.first << " / " << cpu_usage.second; + } + json["cpu_usage"] = ss.str(); + } + { + std::stringstream ss; + ss << "memory_usage is " << gs::memory_to_mb_str(mem_usage.first) << " / " + << gs::memory_to_mb_str(mem_usage.second); + json["memory_usage"] = ss.str(); + } + return seastar::make_ready_future(json.dump()); +} + +} // namespace server \ No newline at end of file diff --git a/flex/engines/http_server/actor/admin_actor.act.h b/flex/engines/http_server/actor/admin_actor.act.h new file mode 100644 index 000000000000..8c70ee528cb2 --- /dev/null +++ b/flex/engines/http_server/actor/admin_actor.act.h @@ -0,0 +1,70 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_HTTP_SERVER_ACTOR_ADMIN_ACT_H_ +#define ENGINES_HTTP_SERVER_ACTOR_ADMIN_ACT_H_ + +#include "flex/engines/http_server/types.h" +#include "flex/engines/http_server/service/hqps_service.h" +#include "flex/engines/graph_db/database/graph_db.h" +#include + +#include +#include + +namespace server { + +class ANNOTATION(actor:impl) admin_actor : public hiactor::actor { + public: + admin_actor(hiactor::actor_base* exec_ctx, const hiactor::byte_t* addr); + ~admin_actor() override; + + seastar::future ANNOTATION(actor:method) run_create_graph(query_param&& param); + + seastar::future ANNOTATION(actor:method) run_get_graph_schema(query_param&& param); + + seastar::future ANNOTATION(actor:method) run_list_graphs(query_param&& param); + + seastar::future ANNOTATION(actor:method) run_delete_graph(query_param&& param); + + seastar::future ANNOTATION(actor:method) run_graph_loading(graph_management_param&& param); + + seastar::future ANNOTATION(actor:method) start_service(query_param&& param); + + seastar::future ANNOTATION(actor:method) service_status(query_param&& param); + + seastar::future ANNOTATION(actor:method) get_procedure_by_procedure_name(procedure_query_param&& param); + + seastar::future ANNOTATION(actor:method) get_procedures_by_graph_name(query_param&& param); + + seastar::future ANNOTATION(actor:method) create_procedure(create_procedure_query_param&& param); + + seastar::future ANNOTATION(actor:method) delete_procedure(procedure_query_param&& param); + + seastar::future ANNOTATION(actor:method) update_procedure(update_procedure_query_param&& param); + + seastar::future ANNOTATION(actor:method) node_status(query_param&& param); + + // DECLARE_RUN_QUERYS; + /// Declare `do_work` func here, no need to implement. + ACTOR_DO_WORK() + + private: + std::mutex mtx_; +}; + +} // namespace server + +#endif // ENGINES_HTTP_SERVER_ACTOR_ADMIN_ACT_H_ diff --git a/flex/engines/http_server/codegen_actor.act.cc b/flex/engines/http_server/actor/codegen_actor.act.cc similarity index 96% rename from flex/engines/http_server/codegen_actor.act.cc rename to flex/engines/http_server/actor/codegen_actor.act.cc index f39a151606cf..bbc9c2e57267 100644 --- a/flex/engines/http_server/codegen_actor.act.cc +++ b/flex/engines/http_server/actor/codegen_actor.act.cc @@ -13,12 +13,11 @@ * limitations under the License. */ -#include "flex/engines/http_server/codegen_actor.act.h" +#include "flex/engines/http_server/actor/codegen_actor.act.h" #include "flex/engines/graph_db/database/graph_db.h" #include "flex/engines/graph_db/database/graph_db_session.h" #include "flex/engines/http_server/codegen_proxy.h" -#include "flex/engines/http_server/stored_procedure.h" #include diff --git a/flex/engines/http_server/codegen_actor.act.h b/flex/engines/http_server/actor/codegen_actor.act.h similarity index 86% rename from flex/engines/http_server/codegen_actor.act.h rename to flex/engines/http_server/actor/codegen_actor.act.h index d2b45652888c..6222a81db4fd 100644 --- a/flex/engines/http_server/codegen_actor.act.h +++ b/flex/engines/http_server/actor/codegen_actor.act.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef ENGINES_HTTP_SERVER_CODEGEN_ACTOR_ACT_H_ -#define ENGINES_HTTP_SERVER_CODEGEN_ACTOR_ACT_H_ +#ifndef ENGINES_HTTP_SERVER_ACTOR_CODEGEN_ACTOR_ACT_H_ +#define ENGINES_HTTP_SERVER_ACTOR_CODEGEN_ACTOR_ACT_H_ #include "flex/engines/http_server/types.h" @@ -38,6 +38,6 @@ class ANNOTATION(actor:impl) codegen_actor : public hiactor::actor { int32_t your_private_members_ = 0; }; -} // namespace serverP +} // namespace server -#endif // ENGINES_HTTP_SERVER_CODEGEN_ACTOR_ACT_H_ +#endif // ENGINES_HTTP_SERVER_ACTOR_CODEGEN_ACTOR_ACT_H_ diff --git a/flex/engines/http_server/actor/executor.act.cc b/flex/engines/http_server/actor/executor.act.cc new file mode 100644 index 000000000000..449ad7702dc1 --- /dev/null +++ b/flex/engines/http_server/actor/executor.act.cc @@ -0,0 +1,123 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/http_server/actor/executor.act.h" + +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/engines/http_server/codegen_proxy.h" +#include "flex/engines/http_server/workdir_manipulator.h" +#include "nlohmann/json.hpp" + +#include + +namespace server { + +executor::~executor() { + // finalization + // ... +} + +executor::executor(hiactor::actor_base* exec_ctx, const hiactor::byte_t* addr) + : hiactor::actor(exec_ctx, addr) { + set_max_concurrency(1); // set max concurrency for task reentrancy (stateful) + // initialization + // ... +} + +seastar::future executor::run_graph_db_query( + query_param&& param) { + auto ret = gs::GraphDB::get() + .GetSession(hiactor::local_shard_id()) + .Eval(param.content); + if (!ret.ok()) { + LOG(ERROR) << "Eval failed: " << ret.status().error_message(); + return seastar::make_exception_future( + seastar::sstring(ret.status().error_message())); + } + auto result = ret.value(); + seastar::sstring content(result.data(), result.size()); + return seastar::make_ready_future(std::move(content)); +} + +// run_query_for stored_procedure +seastar::future executor::run_hqps_procedure_query( + query_param&& param) { + auto& str = param.content; + const char* str_data = str.data(); + size_t str_length = str.size(); + LOG(INFO) << "Receive pay load: " << str_length << " bytes"; + + query::Query cur_query; + if (!cur_query.ParseFromArray(str_data, str_length)) { + LOG(ERROR) << "Fail to parse query from pay load"; + return seastar::make_ready_future( + seastar::sstring("Fail to parse query from pay load")); + } + + auto ret = gs::GraphDB::get() + .GetSession(hiactor::local_shard_id()) + .EvalHqpsProcedure(cur_query); + if (!ret.ok()) { + LOG(ERROR) << "Eval failed: " << ret.status().error_message(); + return seastar::make_exception_future( + seastar::sstring(ret.status().error_message())); + } + auto result = ret.value(); + if (result.size() < 4) { + return seastar::make_exception_future(seastar::sstring( + "Internal Error, more than 4 bytes should be returned")); + } + seastar::sstring content( + result.data() + 4, + result.size() - 4); // skip 4 bytes, since the first 4 + // bytes is the size of the result + return seastar::make_ready_future(std::move(content)); +} + +seastar::future executor::run_hqps_adhoc_query( + adhoc_result&& param) { + LOG(INFO) << "Run adhoc query"; + // The received query's pay load shoud be able to deserialze to physical plan + // 1. load and run. + auto& content = param.content; + LOG(INFO) << "Okay, try to run the query of lib path: " << content.second + << ", job id: " << content.first + << "local shard id: " << hiactor::local_shard_id(); + // seastar::sstring result = server::load_and_run(content.first, + // content.second); + auto ret = gs::GraphDB::get() + .GetSession(hiactor::local_shard_id()) + .EvalAdhoc(content.second); + if (!ret.ok()) { + LOG(ERROR) << "Eval failed: " << ret.status().error_message(); + return seastar::make_exception_future( + seastar::sstring(ret.status().error_message())); + } + auto ret_value = ret.value(); + LOG(INFO) << "Adhoc query result size: " << ret_value.size(); + if (ret_value.size() < 4) { + return seastar::make_exception_future(seastar::sstring( + "Internal Error, more than 4 bytes should be returned")); + } + + seastar::sstring result( + ret_value.data() + 4, + ret_value.size() - 4); // skip 4 bytes, since the first 4 + // bytes is the size of the result + return seastar::make_ready_future(std::move(result)); +} + +} // namespace server diff --git a/flex/engines/http_server/executor.act.h b/flex/engines/http_server/actor/executor.act.h similarity index 90% rename from flex/engines/http_server/executor.act.h rename to flex/engines/http_server/actor/executor.act.h index 0d67e3033866..e3e504415440 100644 --- a/flex/engines/http_server/executor.act.h +++ b/flex/engines/http_server/actor/executor.act.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef ENGINES_HTTP_SERVER_EXECUTOR_ACT_H_ -#define ENGINES_HTTP_SERVER_EXECUTOR_ACT_H_ +#ifndef ENGINES_HTTP_SERVER_ACTOR_EXECUTOR_ACT_H_ +#define ENGINES_HTTP_SERVER_ACTOR_EXECUTOR_ACT_H_ #include "flex/engines/http_server/types.h" @@ -44,4 +44,4 @@ class ANNOTATION(actor:impl) executor : public hiactor::actor { } // namespace server -#endif // ENGINES_HTTP_SERVER_EXECUTOR_ACT_H_ +#endif // ENGINES_HTTP_SERVER_ACTOR_EXECUTOR_ACT_H_ diff --git a/flex/engines/http_server/codegen_proxy.cc b/flex/engines/http_server/codegen_proxy.cc index 3029d5d7dd0d..0db86a203dfa 100644 --- a/flex/engines/http_server/codegen_proxy.cc +++ b/flex/engines/http_server/codegen_proxy.cc @@ -111,7 +111,8 @@ seastar::future CodegenProxy::call_codegen_cmd( } std::string expected_res_lib_path = work_dir + "/lib" + query_name + ".so"; - return call_codegen_cmd_impl(plan_path, query_name, work_dir) + return CallCodegenCmd(plan_path, query_name, work_dir, work_dir, + compiler_graph_schema_, ir_compiler_prop_, codegen_bin_) .then([this, next_job_id, expected_res_lib_path](int codegen_res) { if (codegen_res != 0 || !std::filesystem::exists(expected_res_lib_path)) { @@ -151,16 +152,19 @@ CodegenProxy::get_res_lib_path_from_cache(int32_t next_job_id) { } } -seastar::future CodegenProxy::call_codegen_cmd_impl( +seastar::future CodegenProxy::CallCodegenCmd( const std::string& plan_path, const std::string& query_name, - const std::string& work_dir) { + const std::string& work_dir, const std::string& output_dir, + const std::string& graph_schema_path, const std::string& engine_config, + const std::string& codegen_bin) { // TODO: different suffix for different platform - std::string cmd = codegen_bin_ + " -e=hqps " + " -i=" + plan_path + - " -w=" + work_dir + " --ir_conf=" + ir_compiler_prop_ + - " --graph_schema_path=" + compiler_graph_schema_; + std::string cmd = codegen_bin + " -e=hqps " + " -i=" + plan_path + + " -o=" + output_dir + " --procedure_name=" + query_name + + " -w=" + work_dir + " --ir_conf=" + engine_config + + " --graph_schema_path=" + graph_schema_path; LOG(INFO) << "Start call codegen cmd: [" << cmd << "]"; - return hiactor::thread_resource_pool::submit_work([this, cmd] { + return hiactor::thread_resource_pool::submit_work([cmd] { auto res = std::system(cmd.c_str()); LOG(INFO) << "Codegen cmd: [" << cmd << "] return: " << res; return res; diff --git a/flex/engines/http_server/codegen_proxy.h b/flex/engines/http_server/codegen_proxy.h index b2421d5cbbb7..9677cecd3283 100644 --- a/flex/engines/http_server/codegen_proxy.h +++ b/flex/engines/http_server/codegen_proxy.h @@ -57,14 +57,13 @@ struct StoredProcedureLibMeta { class CodegenProxy { public: static CodegenProxy& get(); + static constexpr const char* DEFAULT_CODEGEN_DIR = "/tmp/codegen/"; CodegenProxy(); ~CodegenProxy(); bool Initialized(); - // the last two params are needed temporally, should be remove after all - // configuration are merged void Init(std::string working_dir, std::string codegen_bin, std::string ir_compiler_prop, std::string compiler_graph_schema); @@ -80,13 +79,15 @@ class CodegenProxy { seastar::future> DoGen( const physical::PhysicalPlan& plan); + static seastar::future CallCodegenCmd( + const std::string& plan_path, const std::string& query_name, + const std::string& work_dir, const std::string& output_dir, + const std::string& graph_schema_path, const std::string& engine_config, + const std::string& codegen_bin); + private: seastar::future call_codegen_cmd(const physical::PhysicalPlan& plan); - seastar::future call_codegen_cmd_impl(const std::string& plan_path, - const std::string& query_name, - const std::string& work_dir); - seastar::future> get_res_lib_path_from_cache( int32_t job_id); diff --git a/flex/engines/http_server/executor.act.cc b/flex/engines/http_server/executor.act.cc deleted file mode 100644 index 917fd0270b20..000000000000 --- a/flex/engines/http_server/executor.act.cc +++ /dev/null @@ -1,87 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "flex/engines/http_server/executor.act.h" - -#include "flex/engines/graph_db/database/graph_db.h" -#include "flex/engines/graph_db/database/graph_db_session.h" -#include "flex/engines/http_server/codegen_proxy.h" -#include "flex/engines/http_server/stored_procedure.h" - -#include - -namespace server { - -executor::~executor() { - // finalization - // ... -} - -executor::executor(hiactor::actor_base* exec_ctx, const hiactor::byte_t* addr) - : hiactor::actor(exec_ctx, addr) { - set_max_concurrency(1); // set max concurrency for task reentrancy (stateful) - // initialization - // ... -} - -seastar::future executor::run_graph_db_query( - query_param&& param) { - auto ret = gs::GraphDB::get() - .GetSession(hiactor::local_shard_id()) - .Eval(param.content); - seastar::sstring content(ret.data(), ret.size()); - return seastar::make_ready_future(std::move(content)); -} - -// run_query_for stored_procedure -seastar::future executor::run_hqps_procedure_query( - query_param&& param) { - auto& str = param.content; - const char* str_data = str.data(); - size_t str_length = str.size(); - LOG(INFO) << "Receive pay load: " << str_length << " bytes"; - - query::Query cur_query; - { - CHECK(cur_query.ParseFromArray(str.data(), str.size())); - LOG(INFO) << "Parse query: " << cur_query.DebugString(); - } - auto& store_procedure_manager = server::StoredProcedureManager::get(); - return store_procedure_manager.Query(cur_query).then( - [&cur_query](results::CollectiveResults&& hqps_result) { - LOG(INFO) << "Finish running query: " << cur_query.DebugString(); - LOG(INFO) << "Query results" << hqps_result.DebugString(); - - auto tem_str = hqps_result.SerializeAsString(); - - seastar::sstring content(tem_str.data(), tem_str.size()); - return seastar::make_ready_future(std::move(content)); - }); -} - -seastar::future executor::run_hqps_adhoc_query( - adhoc_result&& param) { - LOG(INFO) << "Run adhoc query"; - // The received query's pay load shoud be able to deserialze to physical plan - // 1. load and run. - auto& content = param.content; - LOG(INFO) << "Okay, try to run the query of lib path: " << content.second - << ", job id: " << content.first - << "local shard id: " << hiactor::local_shard_id(); - seastar::sstring result = server::load_and_run(content.first, content.second); - return seastar::make_ready_future(std::move(result)); -} - -} // namespace server diff --git a/flex/engines/http_server/handler/admin_http_handler.cc b/flex/engines/http_server/handler/admin_http_handler.cc new file mode 100644 index 000000000000..99823bccb4b2 --- /dev/null +++ b/flex/engines/http_server/handler/admin_http_handler.cc @@ -0,0 +1,665 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/http_server/handler/admin_http_handler.h" +#include "flex/engines/http_server/executor_group.actg.h" +#include "flex/engines/http_server/options.h" + +#include +#include +#include +#include "flex/engines/http_server/generated/actor/admin_actor_ref.act.autogen.h" +#include "flex/engines/http_server/types.h" +#include "flex/engines/http_server/workdir_manipulator.h" + +#include + +namespace server { + +/** + * Handle all request for graph management. + */ +class admin_http_graph_handler_impl : public seastar::httpd::handler_base { + public: + admin_http_graph_handler_impl(uint32_t group_id, uint32_t shard_concurrency) + : shard_concurrency_(shard_concurrency), executor_idx_(0) { + admin_actor_refs_.reserve(shard_concurrency_); + hiactor::scope_builder builder; + builder.set_shard(hiactor::local_shard_id()) + .enter_sub_scope(hiactor::scope(0)) + .enter_sub_scope(hiactor::scope(group_id)); + for (unsigned i = 0; i < shard_concurrency_; ++i) { + admin_actor_refs_.emplace_back(builder.build_ref(i)); + } + } + ~admin_http_graph_handler_impl() override = default; + + seastar::future> handle( + const seastar::sstring& path, + std::unique_ptr req, + std::unique_ptr rep) override { + auto dst_executor = executor_idx_; + + executor_idx_ = (executor_idx_ + 1) % shard_concurrency_; + LOG(INFO) << "Handling path:" << path << ", method: " << req->_method; + auto& method = req->_method; + if (method == "POST") { + if (path.find("dataloading") != seastar::sstring::npos) { + LOG(INFO) << "Route to loading graph"; + if (!req->param.exists("graph_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name not exists")); + } else { + auto graph_name = req->param.at("graph_name"); + LOG(INFO) << "Graph name: " << graph_name; + auto pair = std::make_pair(graph_name, std::move(req->content)); + return admin_actor_refs_[dst_executor] + .run_graph_loading(graph_management_param{std::move(pair)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + } else { + LOG(INFO) << "Route to creating graph"; + return admin_actor_refs_[dst_executor] + .run_create_graph(query_param{std::move(req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + } else if (method == "GET") { + if (req->param.exists("graph_name") && + path.find("schema") != seastar::sstring::npos) { + auto graph_name = req->param.at("graph_name"); + return admin_actor_refs_[dst_executor] + .run_get_graph_schema(query_param{std::move(graph_name)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else { + return admin_actor_refs_[dst_executor] + .run_list_graphs(query_param{std::move(req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + } else if (method == "DELETE") { + if (!req->param.exists("graph_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name not given")); + } + auto graph_name = req->param.at("graph_name"); + return admin_actor_refs_[dst_executor] + .run_delete_graph(query_param{std::move(graph_name)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("Unsupported method" + method)); + } + } + + private: + const uint32_t shard_concurrency_; + uint32_t executor_idx_; + std::vector admin_actor_refs_; +}; + +class admin_http_procedure_handler_impl : public seastar::httpd::handler_base { + public: + admin_http_procedure_handler_impl(uint32_t group_id, + uint32_t shard_concurrency) + : shard_concurrency_(shard_concurrency), executor_idx_(0) { + admin_actor_refs_.reserve(shard_concurrency_); + hiactor::scope_builder builder; + builder.set_shard(hiactor::local_shard_id()) + .enter_sub_scope(hiactor::scope(0)) + .enter_sub_scope(hiactor::scope(group_id)); + for (unsigned i = 0; i < shard_concurrency_; ++i) { + admin_actor_refs_.emplace_back(builder.build_ref(i)); + } + } + ~admin_http_procedure_handler_impl() override = default; + + seastar::future> handle( + const seastar::sstring& path, + std::unique_ptr req, + std::unique_ptr rep) override { + auto dst_executor = executor_idx_; + + executor_idx_ = (executor_idx_ + 1) % shard_concurrency_; + LOG(INFO) << "Handling path:" << path << ", method: " << req->_method; + // LOG(INFO) << "Graph_name:" << req->param.at("graph_name"); + if (req->_method == "GET") { + // get graph_name param + if (!req->param.exists("graph_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name not exists")); + } + auto graph_name = req->param.at("graph_name"); + // remove / from the graph_name + graph_name.erase(std::remove(graph_name.begin(), graph_name.end(), '/'), + graph_name.end()); + if (req->param.exists("procedure_name")) { + // Get the procedures + auto procedure_name = req->param.at("procedure_name"); + // remove / from the procedure_name + procedure_name.erase( + std::remove(procedure_name.begin(), procedure_name.end(), '/'), + procedure_name.end()); + + LOG(INFO) << "Get procedure for: " << graph_name << ", " + << procedure_name; + auto pair = std::make_pair(graph_name, procedure_name); + return admin_actor_refs_[dst_executor] + .get_procedure_by_procedure_name( + procedure_query_param{std::move(pair)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else { + // get all procedures. + LOG(INFO) << "Get all procedures for: " << graph_name; + return admin_actor_refs_[dst_executor] + .get_procedures_by_graph_name(query_param{std::move(graph_name)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + } else if (req->_method == "POST") { + if (!req->param.exists("graph_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name not given")); + } + auto graph_name = req->param.at("graph_name"); + // remove / from the graph_name + graph_name.erase(std::remove(graph_name.begin(), graph_name.end(), '/'), + graph_name.end()); + LOG(INFO) << "Creating procedure for: " << graph_name; + return admin_actor_refs_[dst_executor] + .create_procedure(create_procedure_query_param{ + std::make_pair(graph_name, std::move(req->content))}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else if (req->_method == "DELETE") { + // delete must give graph_name and procedure_name + if (!req->param.exists("graph_name") || + !req->param.exists("procedure_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name or procedure_name not given: ")); + } + auto graph_name = req->param.at("graph_name"); + graph_name.erase(std::remove(graph_name.begin(), graph_name.end(), '/'), + graph_name.end()); + auto procedure_name = req->param.at("procedure_name"); + procedure_name.erase( + std::remove(procedure_name.begin(), procedure_name.end(), '/'), + procedure_name.end()); + LOG(INFO) << "Deleting procedure for: " << graph_name << ", " + << procedure_name; + return admin_actor_refs_[dst_executor] + .delete_procedure( + procedure_query_param{std::make_pair(graph_name, procedure_name)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else if (req->_method == "PUT") { + if (!req->param.exists("graph_name") || + !req->param.exists("procedure_name")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("graph_name or procedure_name not given: ")); + } + auto graph_name = req->param.at("graph_name"); + graph_name.erase(std::remove(graph_name.begin(), graph_name.end(), '/'), + graph_name.end()); + auto procedure_name = req->param.at("procedure_name"); + procedure_name.erase( + std::remove(procedure_name.begin(), procedure_name.end(), '/'), + procedure_name.end()); + LOG(INFO) << "Update procedure for: " << graph_name << ", " + << procedure_name; + return admin_actor_refs_[dst_executor] + .update_procedure(update_procedure_query_param{ + std::make_tuple(graph_name, procedure_name, req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("Unsupported method" + req->_method)); + } + } + + private: + const uint32_t shard_concurrency_; + uint32_t executor_idx_; + std::vector admin_actor_refs_; +}; + +// Handling request for node and service management +class admin_http_service_handler_impl : public seastar::httpd::handler_base { + public: + admin_http_service_handler_impl(uint32_t group_id, uint32_t shard_concurrency) + : shard_concurrency_(shard_concurrency), executor_idx_(0) { + admin_actor_refs_.reserve(shard_concurrency_); + hiactor::scope_builder builder; + builder.set_shard(hiactor::local_shard_id()) + .enter_sub_scope(hiactor::scope(0)) + .enter_sub_scope(hiactor::scope(group_id)); + for (unsigned i = 0; i < shard_concurrency_; ++i) { + admin_actor_refs_.emplace_back(builder.build_ref(i)); + } + } + ~admin_http_service_handler_impl() override = default; + + seastar::future> handle( + const seastar::sstring& path, + std::unique_ptr req, + std::unique_ptr rep) override { + auto dst_executor = executor_idx_; + + executor_idx_ = (executor_idx_ + 1) % shard_concurrency_; + LOG(INFO) << "Handling path:" << path << ", method: " << req->_method; + auto& method = req->_method; + if (method == "POST") { + // Then param[action] should exists + if (!req->param.exists("action")) { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("action is expected for /v1/service/")); + } + auto action = req->param.at("action"); + LOG(INFO) << "POST with action: " << action; + // Remove / from the action + action.erase(std::remove(action.begin(), action.end(), '/'), + action.end()); + + if (action == "start" || action == "restart") { + return admin_actor_refs_[dst_executor] + .start_service(query_param{std::move(req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>( + fut.get_exception()); + } + auto result = fut.get0(); + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else if (action == "stop") { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("Stopping service not supported.")); + } else { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("Unsupported action: " + action)); + } + } else { + // get status + LOG(INFO) << "GET with action: status"; + return admin_actor_refs_[dst_executor] + .service_status(query_param{std::move(req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + LOG(INFO) << "Service status: " << result.content; + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + } + + private: + const uint32_t shard_concurrency_; + uint32_t executor_idx_; + std::vector admin_actor_refs_; +}; + +class admin_http_node_handler_impl : public seastar::httpd::handler_base { + public: + admin_http_node_handler_impl(uint32_t group_id, uint32_t shard_concurrency) + : shard_concurrency_(shard_concurrency), executor_idx_(0) { + admin_actor_refs_.reserve(shard_concurrency_); + hiactor::scope_builder builder; + builder.set_shard(hiactor::local_shard_id()) + .enter_sub_scope(hiactor::scope(0)) + .enter_sub_scope(hiactor::scope(group_id)); + for (unsigned i = 0; i < shard_concurrency_; ++i) { + admin_actor_refs_.emplace_back(builder.build_ref(i)); + } + } + ~admin_http_node_handler_impl() override = default; + + seastar::future> handle( + const seastar::sstring& path, + std::unique_ptr req, + std::unique_ptr rep) override { + auto dst_executor = executor_idx_; + + executor_idx_ = (executor_idx_ + 1) % shard_concurrency_; + LOG(INFO) << "Handling path:" << path << ", method: " << req->_method; + auto& method = req->_method; + if (method == "GET") { + LOG(INFO) << "GET with action: status"; + return admin_actor_refs_[dst_executor] + .node_status(query_param{std::move(req->content)}) + .then_wrapped([rep = std::move(rep)]( + seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + return seastar::make_exception_future< + std::unique_ptr>(fut.get_exception()); + } + auto result = fut.get0(); + LOG(INFO) << "Node status: " << result.content; + rep->write_body("application/json", std::move(result.content)); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } else { + return seastar::make_exception_future< + std::unique_ptr>( + std::runtime_error("Unsupported method" + method)); + } + } + + private: + const uint32_t shard_concurrency_; + uint32_t executor_idx_; + std::vector admin_actor_refs_; +}; + +admin_http_handler::admin_http_handler(uint16_t http_port) + : http_port_(http_port) {} + +void admin_http_handler::start() { + auto fut = seastar::alien::submit_to( + *seastar::alien::internal::default_instance, 0, [this] { + return server_.start() + .then([this] { return set_routes(); }) + .then([this] { return server_.listen(http_port_); }) + .then([this] { + fmt::print("HQPS admin http handler is listening on port {} ...\n", + http_port_); + }); + }); + fut.wait(); +} + +void admin_http_handler::stop() { + auto fut = + seastar::alien::submit_to(*seastar::alien::internal::default_instance, 0, + [this] { return server_.stop(); }); + fut.wait(); +} + +seastar::future<> admin_http_handler::set_routes() { + return server_.set_routes([this](seastar::httpd::routes& r) { + auto admin_graph_handler = new admin_http_graph_handler_impl( + interactive_admin_group_id, shard_admin_graph_concurrency); + + auto procedures_handler = new admin_http_procedure_handler_impl( + interactive_admin_group_id, shard_admin_procedure_concurrency); + + auto service_handler = new admin_http_service_handler_impl( + interactive_admin_group_id, shard_admin_service_concurrency); + + auto node_handler = new admin_http_node_handler_impl( + interactive_admin_group_id, shard_admin_node_concurrency); + + ////Procedure management /// + { + auto match_rule = new seastar::httpd::match_rule(procedures_handler); + match_rule->add_str("/v1/graph") + .add_param("graph_name") + .add_str("/procedure"); + // Get All procedures + r.add(match_rule, seastar::httpd::operation_type::GET); + // Create a new procedure + r.add(match_rule, seastar::httpd::operation_type::POST); + } + { + // Each procedure's handling + auto match_rule = new seastar::httpd::match_rule(procedures_handler); + match_rule->add_str("/v1/graph") + .add_param("graph_name") + .add_str("/procedure") + .add_param("procedure_name"); + // Get a procedure + r.add(match_rule, seastar::httpd::operation_type::GET); + // Delete a procedure + r.add(match_rule, seastar::httpd::operation_type::DELETE); + // Update a procedure + r.add(match_rule, seastar::httpd::operation_type::PUT); + } + + // List all graphs. + r.add(seastar::httpd::operation_type::GET, seastar::httpd::url("/v1/graph"), + admin_graph_handler); + // Create a new Graph + r.add(seastar::httpd::operation_type::POST, + seastar::httpd::url("/v1/graph"), admin_graph_handler); + + // Delete a graph + r.add(seastar::httpd::operation_type::DELETE, + seastar::httpd::url("/v1/graph").remainder("graph_name"), + admin_graph_handler); + + { // load data to graph + auto match_rule = new seastar::httpd::match_rule(admin_graph_handler); + match_rule->add_str("/v1/graph") + .add_param("graph_name") + .add_str("/dataloading"); + r.add(match_rule, seastar::httpd::operation_type::POST); + } + { // Get Graph Schema + auto match_rule = new seastar::httpd::match_rule(admin_graph_handler); + match_rule->add_str("/v1/graph") + .add_param("graph_name") + .add_str("/schema"); + r.add(match_rule, seastar::httpd::operation_type::GET); + } + + { + // Node and service management + r.add(seastar::httpd::operation_type::GET, + seastar::httpd::url("/v1/node/status"), node_handler); + + auto match_rule = new seastar::httpd::match_rule(service_handler); + match_rule->add_str("/v1/service").add_param("action"); + r.add(match_rule, seastar::httpd::operation_type::POST); + + r.add(seastar::httpd::operation_type::GET, + seastar::httpd::url("/v1/service/status"), service_handler); + } + + { + seastar::httpd::parameters params; + auto test_handler = r.get_handler(seastar::httpd::operation_type::POST, + "/v1/graph/abc/dataloading", params); + CHECK(test_handler); + CHECK(params.exists("graph_name")); + CHECK(params.at("graph_name") == "/abc") << params.at("graph_name"); + } + + { + seastar::httpd::parameters params; + auto test_handler = r.get_handler(seastar::httpd::operation_type::GET, + "/v1/graph/abc/schema", params); + CHECK(test_handler); + CHECK(params.exists("graph_name")); + CHECK(params.at("graph_name") == "/abc") << params.at("graph_name"); + } + + { + seastar::httpd::parameters params; + auto test_handler = r.get_handler(seastar::httpd::operation_type::GET, + "/v1/graph/abc/procedure", params); + CHECK(test_handler); + CHECK(params.exists("graph_name")); + CHECK(params.at("graph_name") == "/abc") << params.at("graph_name"); + } + + { + seastar::httpd::parameters params; + auto test_handler = r.get_handler(seastar::httpd::operation_type::POST, + "/v1/graph/abc/procedure", params); + CHECK(test_handler); + CHECK(params.exists("graph_name")); + CHECK(params.at("graph_name") == "/abc") << params.at("graph_name"); + } + + { + seastar::httpd::parameters params; + auto test_handler = + r.get_handler(seastar::httpd::operation_type::GET, + "/v1/graph/abc/procedure/proce1", params); + CHECK(test_handler); + CHECK(params.exists("graph_name")); + CHECK(params.at("graph_name") == "/abc") << params.at("graph_name"); + CHECK(params.exists("procedure_name")); + CHECK(params.at("procedure_name") == "/proce1") + << params.at("procedure_name"); + params.clear(); + test_handler = r.get_handler(seastar::httpd::operation_type::DELETE, + "/v1/graph/abc/procedure/proce1", params); + CHECK(test_handler); + test_handler = r.get_handler(seastar::httpd::operation_type::PUT, + "/v1/graph/abc/procedure/proce1", params); + CHECK(test_handler); + } + + return seastar::make_ready_future<>(); + }); +} + +} // namespace server diff --git a/flex/engines/http_server/handler/admin_http_handler.h b/flex/engines/http_server/handler/admin_http_handler.h new file mode 100644 index 000000000000..3d5c3f92a42a --- /dev/null +++ b/flex/engines/http_server/handler/admin_http_handler.h @@ -0,0 +1,45 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_HTTP_SERVER_HANDLER_ADMIN_HTTP_HANDLER_H_ +#define ENGINES_HTTP_SERVER_HANDLER_ADMIN_HTTP_HANDLER_H_ + +#include +#include +#include +#include "flex/engines/http_server/types.h" +#include "flex/utils/service_utils.h" + +namespace server { + +class InteractiveAdminService; +class admin_http_handler { + public: + admin_http_handler(uint16_t http_port); + + void start(); + void stop(); + + private: + seastar::future<> set_routes(); + + private: + const uint16_t http_port_; + seastar::httpd::http_server_control server_; +}; + +} // namespace server + +#endif // ENGINES_HTTP_SERVER_HANDLER_ADMIN_HTTP_HANDLER_H_ diff --git a/flex/engines/http_server/graph_db_http_handler.cc b/flex/engines/http_server/handler/graph_db_http_handler.cc similarity index 97% rename from flex/engines/http_server/graph_db_http_handler.cc rename to flex/engines/http_server/handler/graph_db_http_handler.cc index 0514160aac43..b5aac790c298 100644 --- a/flex/engines/http_server/graph_db_http_handler.cc +++ b/flex/engines/http_server/handler/graph_db_http_handler.cc @@ -14,13 +14,13 @@ */ #include "flex/engines/http_server/executor_group.actg.h" -#include "flex/engines/http_server/graph_db_service.h" #include "flex/engines/http_server/options.h" +#include "flex/engines/http_server/service/graph_db_service.h" #include #include #include -#include "flex/engines/http_server/generated/executor_ref.act.autogen.h" +#include "flex/engines/http_server/generated/actor/executor_ref.act.autogen.h" #include "flex/engines/http_server/types.h" namespace server { diff --git a/flex/engines/http_server/graph_db_http_handler.h b/flex/engines/http_server/handler/graph_db_http_handler.h similarity index 83% rename from flex/engines/http_server/graph_db_http_handler.h rename to flex/engines/http_server/handler/graph_db_http_handler.h index 0f854f51c48d..64e03913280b 100644 --- a/flex/engines/http_server/graph_db_http_handler.h +++ b/flex/engines/http_server/handler/graph_db_http_handler.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef ENGINES_HTTP_SERVER_GRAPH_DB_HTTP_HANDLER_H_ -#define ENGINES_HTTP_SERVER_GRAPH_DB_HTTP_HANDLER_H_ +#ifndef ENGINES_HTTP_SERVER_HANDLER_GRAPH_DB_HTTP_HANDLER_H_ +#define ENGINES_HTTP_SERVER_HANDLER_GRAPH_DB_HTTP_HANDLER_H_ #include @@ -37,4 +37,4 @@ class graph_db_http_handler { } // namespace server -#endif // ENGINES_HTTP_SERVER_GRAPH_DB_HTTP_HANDLER_H_ +#endif // ENGINES_HTTP_SERVER_HANDLER_GRAPH_DB_HTTP_HANDLER_H_ diff --git a/flex/engines/http_server/hqps_http_handler.cc b/flex/engines/http_server/handler/hqps_http_handler.cc similarity index 87% rename from flex/engines/http_server/hqps_http_handler.cc rename to flex/engines/http_server/handler/hqps_http_handler.cc index 40a4897117e4..d4baefa0f667 100644 --- a/flex/engines/http_server/hqps_http_handler.cc +++ b/flex/engines/http_server/handler/hqps_http_handler.cc @@ -13,14 +13,14 @@ * limitations under the License. */ #include "flex/engines/http_server/executor_group.actg.h" -#include "flex/engines/http_server/hqps_service.h" #include "flex/engines/http_server/options.h" +#include "flex/engines/http_server/service/hqps_service.h" #include #include #include -#include "flex/engines/http_server/generated/codegen_actor_ref.act.autogen.h" -#include "flex/engines/http_server/generated/executor_ref.act.autogen.h" +#include "flex/engines/http_server/generated/actor/codegen_actor_ref.act.autogen.h" +#include "flex/engines/http_server/generated/actor/executor_ref.act.autogen.h" #include "flex/engines/http_server/types.h" namespace server { @@ -152,9 +152,7 @@ class hqps_exit_handler : public seastar::httpd::handler_base { std::unique_ptr req, std::unique_ptr rep) override { HQPSService::get().set_exit_state(); - rep->write_body( - "bin", - seastar::sstring{"The ldbc snb interactive service is exiting ..."}); + rep->write_body("bin", seastar::sstring{"HQPS service is exiting ..."}); return seastar::make_ready_future>( std::move(rep)); } @@ -163,6 +161,10 @@ class hqps_exit_handler : public seastar::httpd::handler_base { hqps_http_handler::hqps_http_handler(uint16_t http_port) : http_port_(http_port) {} +uint16_t hqps_http_handler::get_port() const { return http_port_; } + +bool hqps_http_handler::is_running() const { return running_.load(); } + void hqps_http_handler::start() { auto fut = seastar::alien::submit_to( *seastar::alien::internal::default_instance, 0, [this] { @@ -171,26 +173,35 @@ void hqps_http_handler::start() { .then([this] { return server_.listen(http_port_); }) .then([this] { fmt::print( - "Ldbc snb interactive http handler is listening on port {} " + "HQPS Query http handler is listening on port {} " "...\n", http_port_); }); }); fut.wait(); + // update running state + running_.store(true); } void hqps_http_handler::stop() { - auto fut = - seastar::alien::submit_to(*seastar::alien::internal::default_instance, 0, - [this] { return server_.stop(); }); + auto fut = seastar::alien::submit_to( + *seastar::alien::internal::default_instance, 0, [this] { + LOG(INFO) << "Stopping HQPS http handler ..."; + return server_.stop(); + }); fut.wait(); + // update running state + running_.store(false); } seastar::future<> hqps_http_handler::set_routes() { return server_.set_routes([this](seastar::httpd::routes& r) { + auto procedure_handler = + new hqps_ic_handler(ic_query_group_id, shard_query_concurrency); + r.add(seastar::httpd::operation_type::POST, + seastar::httpd::url("/interactive/query"), procedure_handler); r.add(seastar::httpd::operation_type::POST, - seastar::httpd::url("/interactive/query"), - new hqps_ic_handler(ic_query_group_id, shard_query_concurrency)); + seastar::httpd::url("/v1/query"), procedure_handler); r.add(seastar::httpd::operation_type::POST, seastar::httpd::url("/interactive/adhoc_query"), new hqps_adhoc_query_handler(ic_adhoc_group_id, codegen_group_id, diff --git a/flex/engines/http_server/hqps_http_handler.h b/flex/engines/http_server/handler/hqps_http_handler.h similarity index 77% rename from flex/engines/http_server/hqps_http_handler.h rename to flex/engines/http_server/handler/hqps_http_handler.h index ce2ea7eeb1a2..dcc0af897693 100644 --- a/flex/engines/http_server/hqps_http_handler.h +++ b/flex/engines/http_server/handler/hqps_http_handler.h @@ -12,8 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef ENGINES_HTTP_SERVER_HQPS_HTTP_HANDLER_H_ -#define ENGINES_HTTP_SERVER_HQPS_HTTP_HANDLER_H_ +#ifndef ENGINES_HTTP_SERVER_HANDLER_HQPS_HTTP_HANDLER_H_ +#define ENGINES_HTTP_SERVER_HANDLER_HQPS_HTTP_HANDLER_H_ #include @@ -26,14 +26,19 @@ class hqps_http_handler { void start(); void stop(); + uint16_t get_port() const; + + bool is_running() const; + private: seastar::future<> set_routes(); private: const uint16_t http_port_; seastar::httpd::http_server_control server_; + std::atomic running_{false}; }; } // namespace server -#endif // ENGINES_HTTP_SERVER_HQPS_HTTP_HANDLER_H_ \ No newline at end of file +#endif // ENGINES_HTTP_SERVER_HANDLER_HQPS_HTTP_HANDLER_H_ diff --git a/flex/engines/http_server/hqps_service.cc b/flex/engines/http_server/hqps_service.cc deleted file mode 100644 index e503eea293a0..000000000000 --- a/flex/engines/http_server/hqps_service.cc +++ /dev/null @@ -1,50 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "flex/engines/http_server/hqps_service.h" -#include "flex/engines/http_server/options.h" -namespace server { - -void HQPSService::init(uint32_t num_shards, uint16_t http_port, bool dpdk_mode, - bool enable_thread_resource_pool, - unsigned external_thread_num) { - actor_sys_ = std::make_unique( - num_shards, dpdk_mode, enable_thread_resource_pool, external_thread_num); - http_hdl_ = std::make_unique(http_port); -} - -HQPSService::~HQPSService() { - if (actor_sys_) { - actor_sys_->terminate(); - } -} - -void HQPSService::run_and_wait_for_exit() { - if (!actor_sys_ || !http_hdl_) { - std::cerr << "High QPS service has not been inited!" << std::endl; - return; - } - actor_sys_->launch(); - http_hdl_->start(); - running_.store(true); - while (running_.load(std::memory_order_relaxed)) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - http_hdl_->stop(); - actor_sys_->terminate(); -} - -void HQPSService::set_exit_state() { running_.store(false); } - -} // namespace server diff --git a/flex/engines/http_server/options.cc b/flex/engines/http_server/options.cc index f61df97f1508..3a2274598d9b 100644 --- a/flex/engines/http_server/options.cc +++ b/flex/engines/http_server/options.cc @@ -20,5 +20,9 @@ namespace server { uint32_t shard_query_concurrency = 16; uint32_t shard_update_concurrency = 4; uint32_t shard_adhoc_concurrency = 4; +uint32_t shard_admin_graph_concurrency = 1; +uint32_t shard_admin_procedure_concurrency = 1; +uint32_t shard_admin_node_concurrency = 1; +uint32_t shard_admin_service_concurrency = 1; } // namespace server diff --git a/flex/engines/http_server/options.h b/flex/engines/http_server/options.h index 7fbdde48a700..819937367415 100644 --- a/flex/engines/http_server/options.h +++ b/flex/engines/http_server/options.h @@ -25,10 +25,15 @@ const uint32_t ic_query_group_id = 1; const uint32_t ic_update_group_id = 2; const uint32_t ic_adhoc_group_id = 3; const uint32_t codegen_group_id = 4; +const uint32_t interactive_admin_group_id = 5; extern uint32_t shard_query_concurrency; extern uint32_t shard_update_concurrency; extern uint32_t shard_adhoc_concurrency; +extern uint32_t shard_admin_graph_concurrency; +extern uint32_t shard_admin_node_concurrency; +extern uint32_t shard_admin_service_concurrency; +extern uint32_t shard_admin_procedure_concurrency; } // namespace server diff --git a/flex/engines/http_server/graph_db_service.cc b/flex/engines/http_server/service/graph_db_service.cc similarity index 95% rename from flex/engines/http_server/graph_db_service.cc rename to flex/engines/http_server/service/graph_db_service.cc index c0335c62b482..93530445a35c 100644 --- a/flex/engines/http_server/graph_db_service.cc +++ b/flex/engines/http_server/service/graph_db_service.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "flex/engines/http_server/graph_db_service.h" +#include "flex/engines/http_server/service/graph_db_service.h" #include "flex/engines/http_server/options.h" namespace server { diff --git a/flex/engines/http_server/graph_db_service.h b/flex/engines/http_server/service/graph_db_service.h similarity index 83% rename from flex/engines/http_server/graph_db_service.h rename to flex/engines/http_server/service/graph_db_service.h index c26cfdafde44..903e99f629d3 100644 --- a/flex/engines/http_server/graph_db_service.h +++ b/flex/engines/http_server/service/graph_db_service.h @@ -13,11 +13,11 @@ * limitations under the License. */ -#ifndef ENGINES_HTTP_SERVER_GRAPH_DB_SERVICE_H_ -#define ENGINES_HTTP_SERVER_GRAPH_DB_SERVICE_H_ +#ifndef ENGINES_HTTP_SERVER_SERVICE_GRAPH_DB_SERVICE_H_ +#define ENGINES_HTTP_SERVER_SERVICE_GRAPH_DB_SERVICE_H_ #include "flex/engines/http_server/actor_system.h" -#include "flex/engines/http_server/graph_db_http_handler.h" +#include "flex/engines/http_server/handler/graph_db_http_handler.h" namespace server { @@ -44,4 +44,4 @@ class GraphDBService { } // namespace server -#endif // ENGINES_HTTP_SERVER_GRAPH_DB_SERVICE_H_ +#endif // ENGINES_HTTP_SERVER_SERVICE_GRAPH_DB_SERVICE_H_ diff --git a/flex/engines/http_server/service/hqps_service.cc b/flex/engines/http_server/service/hqps_service.cc new file mode 100644 index 000000000000..1b65ab5d983e --- /dev/null +++ b/flex/engines/http_server/service/hqps_service.cc @@ -0,0 +1,113 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "flex/engines/http_server/service/hqps_service.h" +#include "flex/engines/http_server/options.h" +namespace server { + +const std::string HQPSService::DEFAULT_GRAPH_NAME = "modern_graph"; + +HQPSService& HQPSService::get() { + static HQPSService instance; + return instance; +} + +void HQPSService::init(uint32_t num_shards, uint16_t query_port, bool dpdk_mode, + bool enable_thread_resource_pool, + unsigned external_thread_num) { + if (initialized_.load(std::memory_order_relaxed)) { + std::cerr << "High QPS service has been already initialized!" << std::endl; + return; + } + actor_sys_ = std::make_unique( + num_shards, dpdk_mode, enable_thread_resource_pool, external_thread_num); + query_hdl_ = std::make_unique(query_port); + initialized_.store(true); + gs::init_cpu_usage_watch(); +} + +void HQPSService::init(uint32_t num_shards, uint16_t admin_port, + uint16_t query_port, bool dpdk_mode, + bool enable_thread_resource_pool, + unsigned external_thread_num) { + if (initialized_.load(std::memory_order_relaxed)) { + std::cerr << "High QPS service has been already initialized!" << std::endl; + return; + } + actor_sys_ = std::make_unique( + num_shards, dpdk_mode, enable_thread_resource_pool, external_thread_num); + query_hdl_ = std::make_unique(query_port); + admin_hdl_ = std::make_unique(admin_port); + initialized_.store(true); + gs::init_cpu_usage_watch(); +} + +HQPSService::~HQPSService() { + if (actor_sys_) { + actor_sys_->terminate(); + } +} + +bool HQPSService::is_initialized() const { + return initialized_.load(std::memory_order_relaxed); +} + +bool HQPSService::is_running() const { + return running_.load(std::memory_order_relaxed); +} + +uint16_t HQPSService::get_query_port() const { + if (query_hdl_) { + return query_hdl_->get_port(); + } + return 0; +} + +gs::Result HQPSService::service_status() { + if (!is_initialized()) { + return gs::Result( + gs::StatusCode::OK, "High QPS service has not been inited!", ""); + } + if (!is_running()) { + return gs::Result( + gs::StatusCode::OK, "High QPS service has not been started!", ""); + } + return gs::Result( + seastar::sstring("High QPS service is running ...")); +} + +void HQPSService::run_and_wait_for_exit() { + if (!is_initialized()) { + std::cerr << "High QPS service has not been inited!" << std::endl; + return; + } + actor_sys_->launch(); + query_hdl_->start(); + if (admin_hdl_) { + admin_hdl_->start(); + } + running_.store(true); + while (running_.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + query_hdl_->stop(); + if (admin_hdl_) { + admin_hdl_->stop(); + } + actor_sys_->terminate(); +} + +void HQPSService::set_exit_state() { running_.store(false); } + +} // namespace server diff --git a/flex/engines/http_server/hqps_service.h b/flex/engines/http_server/service/hqps_service.h similarity index 55% rename from flex/engines/http_server/hqps_service.h rename to flex/engines/http_server/service/hqps_service.h index 49e2776294bc..7a011144e915 100644 --- a/flex/engines/http_server/hqps_service.h +++ b/flex/engines/http_server/service/hqps_service.h @@ -17,24 +17,40 @@ #include +#include "flex/engines/graph_db/database/graph_db.h" #include "flex/engines/http_server/actor_system.h" -#include "flex/engines/http_server/hqps_http_handler.h" +#include "flex/engines/http_server/handler/admin_http_handler.h" +#include "flex/engines/http_server/handler/hqps_http_handler.h" +#include "flex/utils/result.h" +#include "flex/utils/service_utils.h" namespace server { class HQPSService { public: - static HQPSService& get() { - static HQPSService instance; - return instance; - } + static const std::string DEFAULT_GRAPH_NAME; + static HQPSService& get(); ~HQPSService(); - // the store procedure contains - void init(uint32_t num_shards, uint16_t http_port, bool dpdk_mode, + // only start the query service. + void init(uint32_t num_shards, uint16_t query_port, bool dpdk_mode, bool enable_thread_resource_pool, unsigned external_thread_num); + // start both admin and query service. + void init(uint32_t num_shards, uint16_t admin_port, uint16_t query_port, + bool dpdk_mode, bool enable_thread_resource_pool, + unsigned external_thread_num); + + bool is_initialized() const; + + bool is_running() const; + + uint16_t get_query_port() const; + + gs::Result service_status(); + void run_and_wait_for_exit(); + void set_exit_state(); private: @@ -42,8 +58,10 @@ class HQPSService { private: std::unique_ptr actor_sys_; - std::unique_ptr http_hdl_; + std::unique_ptr admin_hdl_; + std::unique_ptr query_hdl_; std::atomic running_{false}; + std::atomic initialized_{false}; }; } // namespace server diff --git a/flex/engines/http_server/stored_procedure.cc b/flex/engines/http_server/stored_procedure.cc deleted file mode 100644 index a4601d5d55f0..000000000000 --- a/flex/engines/http_server/stored_procedure.cc +++ /dev/null @@ -1,182 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "flex/engines/http_server/stored_procedure.h" -#include "flex/engines/graph_db/database/graph_db.h" -#include "flex/engines/http_server/generated/executor_ref.act.autogen.h" - -#include -#include -#include - -namespace server { - -void put_argment(gs::Encoder& encoder, const query::Argument& argment) { - auto& value = argment.value(); - auto item_case = value.item_case(); - switch (item_case) { - case common::Value::kI32: - encoder.put_int(value.i32()); - break; - case common::Value::kI64: - encoder.put_long(value.i64()); - break; - case common::Value::kF64: - encoder.put_double(value.f64()); - break; - case common::Value::kStr: - encoder.put_string(value.str()); - break; - default: - LOG(ERROR) << "Not recognizable param type" << static_cast(item_case); - } -} - -// get the handle of the dynamic library, throw error if needed -void* open_lib(const char* lib_path) { - LOG(INFO) << "try to open library: " << lib_path; - void* handle = dlopen(lib_path, RTLD_LAZY); - auto* p_error_msg = dlerror(); - if (p_error_msg) { - LOG(FATAL) << "Fail to open library: " << lib_path - << ", error: " << p_error_msg; - } - LOG(INFO) << "Successfully open library: " << lib_path; - return handle; -} - -void* get_func_ptr(const char* lib_path, void* handle, const char* symbol) { - auto* p_func = dlsym(handle, symbol); - auto* p_error_msg = dlerror(); - if (p_error_msg) { - LOG(FATAL) << "Failed to get symbol " << symbol << " from " << lib_path - << ". Reason: " << std::string(p_error_msg); - } - return p_func; -} - -// close the handle of the dynamic library, throw error if needed -void close_lib(void* handle, const char* lib_path) { - if (handle) { - auto ret = dlclose(handle); - if (ret == 0) { - LOG(INFO) << "Sucessfuly closed library" << lib_path; - } else { - auto* p_error_msg = dlerror(); - if (p_error_msg) { - LOG(FATAL) << "Fail to close library, error: " << p_error_msg; - } - } - } else { - LOG(WARNING) << "Try to close a null handle," << lib_path; - } -} - -std::vector parse_from_multiple_yamls( - const std::string& plugin_dir, - const std::vector& stored_procedure_yamls, - const std::vector& valid_procedure_names) { - std::vector stored_procedures; - for (auto cur_yaml : stored_procedure_yamls) { - LOG(INFO) << "Loading for: " << cur_yaml; - YAML::Node root = YAML::LoadFile(cur_yaml); - if (!root["name"]) { - LOG(ERROR) << "Expect name in pre_installed procedure"; - } else if (!root["library"]) { - LOG(ERROR) << "Expect path in pre_installed procedure"; - } else { - std::string name = root["name"].as(); - if (find(valid_procedure_names.begin(), valid_procedure_names.end(), - name) != valid_procedure_names.end()) { - VLOG(10) << "Find valid procedure: " << name; - std::string path = root["library"].as(); - if (!std::filesystem::exists(path)) { - // in case the path is relative to plugin_dir, prepend plugin_dir - path = plugin_dir + "/" + path; - if (!std::filesystem::exists(path)) { - LOG(ERROR) << "plugin - " << path << " file not found..."; - } else { - stored_procedures.push_back({name, path}); - } - } else { - stored_procedures.push_back({name, path}); - } - } - } - } - return stored_procedures; -} - -std::vector parse_stored_procedures( - const std::string& stored_procedure_yaml) { - std::vector stored_procedures; - YAML::Node root = YAML::LoadFile(stored_procedure_yaml); - if (root["pre_installed"]) { - std::vector installed_got; - if (!get_sequence(root, "pre_installed", installed_got)) { - LOG(ERROR) << "installed_got is not set properly"; - } - for (auto& procedure : installed_got) { - if (!procedure["name"]) { - LOG(ERROR) << "Expect name in pre_installed procedure"; - } else if (!procedure["path"]) { - LOG(ERROR) << "Expect path in pre_installed procedure"; - } else { - std::string name = procedure["name"].as(); - std::string path = procedure["path"].as(); - if (!std::filesystem::exists(path)) { - LOG(ERROR) << "plugin - " << path << " file not found..."; - } else { - stored_procedures.push_back({name, path}); - } - } - } - } else { - LOG(WARNING) << "Expect entry : " << stored_procedure_yaml; - } - return stored_procedures; -} - -std::shared_ptr create_stored_procedure_impl( - int32_t procedure_id, const std::string& procedure_path) { - auto& sess = gs::GraphDB::get().GetSession(hiactor::local_shard_id()); - - gs::MutableCSRInterface graph_store(sess); - - return std::make_shared< - server::CypherStoredProcedure>( - procedure_id, procedure_path, std::move(graph_store), - gs::GraphStoreType::Grape); -} - -std::string load_and_run(int32_t job_id, const std::string& lib_path) { - auto temp_stored_procedure = - server::create_stored_procedure_impl(job_id, lib_path); - LOG(INFO) << "Create stored procedure: " << temp_stored_procedure->ToString(); - std::vector empty; - gs::Decoder input_decoder(empty.data(), empty.size()); - auto res = temp_stored_procedure->Query(input_decoder); - LOG(INFO) << "Finish running"; - VLOG(10) << res.DebugString(); - std::string res_str; - res.SerializeToString(&res_str); - return res_str; -} - -StoredProcedureManager& StoredProcedureManager::get() { - static StoredProcedureManager instance; - return instance; -} - -} // namespace server diff --git a/flex/engines/http_server/stored_procedure.h b/flex/engines/http_server/stored_procedure.h deleted file mode 100644 index 609d9f75dc2e..000000000000 --- a/flex/engines/http_server/stored_procedure.h +++ /dev/null @@ -1,291 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef ENGINES_HTTP_SERVER_STORED_PROCEDURE_H_ -#define ENGINES_HTTP_SERVER_STORED_PROCEDURE_H_ - -#include -#include - -#include -#include -#include -#include -#include -#include "glog/logging.h" - -#include "flex/proto_generated_gie/results.pb.h" -#include "flex/proto_generated_gie/stored_procedure.pb.h" - -#include -#include -#include "flex/engines/graph_db/database/graph_db_session.h" -#include "flex/engines/hqps_db/app/hqps_app_base.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" -#include "flex/utils/app_utils.h" -#include "flex/utils/yaml_utils.h" - -#include - -namespace server { - -std::string load_and_run(int32_t job_id, const std::string& lib_path); - -// get the handle of the dynamic library, throw error if needed -void* open_lib(const char* lib_path); - -void* get_func_ptr(const char* lib_path, void* handle, const char* symbol); - -// close the handle of the dynamic library, throw error if needed -void close_lib(void* handle, const char* lib_path); - -void put_argment(gs::Encoder& encoder, const query::Argument& argment); - -template -bool get_scalar(YAML::Node node, const std::string& key, T& value) { - YAML::Node cur = node[key]; - if (cur && cur.IsScalar()) { - value = cur.as(); - return true; - } - return false; -} - -template -bool get_sequence(YAML::Node node, const std::string& key, - std::vector& seq) { - YAML::Node cur = node[key]; - if (cur && cur.IsSequence()) { - int num = cur.size(); - seq.clear(); - for (int i = 0; i < num; ++i) { - seq.push_back(cur[i].as()); - } - return true; - } - return false; -} -struct StoredProcedureMeta { - std::string name; - std::string path; -}; - -std::vector parse_stored_procedures( - const std::string& stored_procedure_yaml); -std::vector parse_from_multiple_yamls( - const std::string& plugin_dir, - const std::vector& stored_procedure_yamls, - const std::vector& valid_procedure_names); - -enum class StoredProcedureType { - kCypher = 0, - kSut = 1, -}; - -// return a void* ptr with no params -typedef void* CreateAppT(gs::GraphStoreType); - -// return void with void* as input -typedef void DeleteAppT(void*, gs::GraphStoreType); - -// the root interface of stored produce -class BaseStoredProcedure { - public: - BaseStoredProcedure(int32_t procedure_id, std::string procedure_path) - : procedure_id_(procedure_id), - procedure_path_(procedure_path), - dl_handle_(nullptr) { - dl_handle_ = open_lib(procedure_path.c_str()); - CHECK(dl_handle_); - } - virtual ~BaseStoredProcedure() { - LOG(INFO) << "Destructing stored procedure" << ToString(); - } - virtual StoredProcedureType GetType() const = 0; - - virtual results::CollectiveResults Query(gs::Decoder& decoder) const = 0; - - virtual void delete_app() = 0; - - virtual std::string ToString() const { - std::stringstream ss; - ss << "StoredProcedure{" - << "procedure_id: " << procedure_id_ - << "}, {procedure_path: " << procedure_path_ << "}"; - return ss.str(); - } - - int32_t GetProcedureId() const { return procedure_id_; } - std::string GetProcedureName() const { return procedure_path_; } - - protected: - int32_t procedure_id_; - std::string procedure_path_; - void* dl_handle_; -}; - -template -class CypherStoredProcedure; - -// Create StoredProcedure -// Why we extract the function here rather then put it in the class? -// To support ad-hoc query, and reuse code. - -std::shared_ptr create_stored_procedure_impl( - int32_t procedure_id, const std::string& procedure_path); - -class StoredProcedureManager { - public: - static StoredProcedureManager& get(); - StoredProcedureManager() {} - - // expect multiple query.yaml under this directory. - void LoadFromPluginDir( - const std::string& plugin_dir, - const std::vector& valid_procedure_names) { - auto yaml_files = gs::get_yaml_files(plugin_dir); - auto stored_procedures = parse_from_multiple_yamls(plugin_dir, yaml_files, - valid_procedure_names); - CreateStoredProcedures(stored_procedures); - } - - void LoadFromYaml(const std::string& stored_procedure_yaml) { - auto stored_procedures = parse_stored_procedures(stored_procedure_yaml); - CreateStoredProcedures(stored_procedures); - } - - void CreateStoredProcedures( - const std::vector& stored_procedures) { - for (auto i = 0; i < stored_procedures.size(); ++i) { - stored_procedures_.emplace( - stored_procedures[i].name, - server::create_stored_procedure_impl(i, stored_procedures[i].path)); - } - - LOG(INFO) << "Load [" << stored_procedures_.size() << "] stored procedures"; - } - - seastar::future Query( - const query::Query& query_pb) const { - auto query_name = query_pb.query_name().name(); - if (query_name.empty()) { - LOG(ERROR) << "Query name is empty"; - return seastar::make_exception_future( - std::runtime_error("Query name is empty")); - } - auto it = stored_procedures_.find(query_name); - if (it != stored_procedures_.end()) { - // create a decoder to decode the query - std::vector input_buffer; - gs::Encoder input_encoder(input_buffer); - auto& args = query_pb.arguments(); - for (auto i = 0; i < args.size(); ++i) { - auto& arg = args[i]; - LOG(INFO) << "Putting " << i << "th arg" << arg.DebugString(); - put_argment(input_encoder, arg); - } - LOG(INFO) << "Before running " << query_name; - gs::Decoder input_decoder(input_buffer.data(), input_buffer.size()); - auto result = it->second->Query(input_decoder); - return seastar::make_ready_future( - std::move(result)); - } else { - LOG(ERROR) << "No stored procedure with id: " << query_name; - return seastar::make_exception_future( - std::runtime_error("No stored procedure with id: " + query_name)); - } - } - - private: - std::unordered_map> - stored_procedures_; -}; - -// one stored procedure contains one dynamic lib, two function pointer -// one for create app, other for delete app; -template -class CypherStoredProcedure : public BaseStoredProcedure { - public: - static constexpr const char* CREATOR_APP_FUNC_NAME = "CreateApp"; - static constexpr const char* DELETER_APP_FUNC_NAME = "DeleteApp"; - - CypherStoredProcedure(int32_t procedure_id, std::string procedure_path, - GRAPH_TYPE&& graph, gs::GraphStoreType graph_store_type) - : BaseStoredProcedure(procedure_id, procedure_path), - app_ptr_(nullptr), - create_app_ptr_(nullptr), - delete_app_ptr_(nullptr), - graph_(std::move(graph)), - graph_store_type_(graph_store_type) { - LOG(INFO) << "creating stored procedure: v label num: " - << std::to_string( - graph_.GetDBSession().schema().vertex_label_num()); - // get the func_ptr we need for cypher query. - create_app_ptr_ = reinterpret_cast(get_func_ptr( - procedure_path_.c_str(), dl_handle_, CREATOR_APP_FUNC_NAME)); - CHECK(create_app_ptr_); - delete_app_ptr_ = reinterpret_cast(get_func_ptr( - procedure_path_.c_str(), dl_handle_, DELETER_APP_FUNC_NAME)); - CHECK(delete_app_ptr_); - LOG(INFO) << "Successfully get cypher query function pointer"; - app_ptr_ = reinterpret_cast*>( - create_app_ptr_(graph_store_type_)); - CHECK(app_ptr_); - LOG(INFO) << "Successfully create app"; - } - - virtual ~CypherStoredProcedure() { - if (app_ptr_) { - delete_app(); - } - } - - StoredProcedureType GetType() const override { - return StoredProcedureType::kCypher; - } - - results::CollectiveResults Query(gs::Decoder& decoder) const override { - CHECK(app_ptr_); - LOG(INFO) << "Start to query with cypher stored procedure"; - LOG(INFO) << "label num:" - << graph_.GetDBSession().schema().vertex_label_num(); - return app_ptr_->Query(graph_, decoder); - } - - void delete_app() override { - LOG(INFO) << "Start to delete app"; - delete_app_ptr_(static_cast(app_ptr_), graph_store_type_); - LOG(INFO) << "Successfully delete app"; - } - - std::string ToString() const override { - std::stringstream ss; - ss << "CypherStoredProcedure{" - << "procedure_id: " << procedure_id_ - << "}, {procedure_path: " << procedure_path_ << "}"; - return ss.str(); - } - - private: - GRAPH_TYPE graph_; - gs::GraphStoreType graph_store_type_; - gs::HqpsAppBase* app_ptr_; - - // func ptr; - CreateAppT* create_app_ptr_; - DeleteAppT* delete_app_ptr_; -}; -} // namespace server - -#endif // ENGINES_HTTP_SERVER_STORED_PROCEDURE_H_ diff --git a/flex/engines/http_server/types.h b/flex/engines/http_server/types.h index 692670b00291..965fbd7137d3 100644 --- a/flex/engines/http_server/types.h +++ b/flex/engines/http_server/types.h @@ -16,15 +16,18 @@ #ifndef ENGINES_HTTP_SERVER_TYPES_H_ #define ENGINES_HTTP_SERVER_TYPES_H_ +#include #include #include #include +#include "flex/utils/service_utils.h" #include namespace server { using timestamp_t = uint32_t; +using boost_ptree = boost::property_tree::ptree; template ; using query_result = payload; using adhoc_result = payload>; +using admin_query_result = payload>; +// url_path, query_param +using graph_management_param = + payload>; +using procedure_query_param = + payload>; +using create_procedure_query_param = + payload>; +using update_procedure_query_param = + payload>; } // namespace server diff --git a/flex/engines/http_server/workdir_manipulator.cc b/flex/engines/http_server/workdir_manipulator.cc new file mode 100644 index 000000000000..16d198c7737f --- /dev/null +++ b/flex/engines/http_server/workdir_manipulator.cc @@ -0,0 +1,1280 @@ +#include "flex/engines/http_server/workdir_manipulator.h" +#include "flex/engines/http_server/codegen_proxy.h" + +// Write a macro to define the function, to check whether a filed presents in a +// json object. +#define CHECK_JSON_FIELD(json, field) \ + if (!json.contains(field)) { \ + return gs::Result( \ + gs::Status(gs::StatusCode::InValidArgument, \ + "Procedure " + std::string(field) + " is not specified")); \ + } + +namespace server { +std::string WorkDirManipulator::workspace = "."; // default to . + +void WorkDirManipulator::SetWorkspace(const std::string& path) { + workspace = path; +} + +void WorkDirManipulator::SetRunningGraph(const std::string& name) { + // clear the old RUNNING_GRAPH_FILE_NAME, and write the new one. + auto running_graph_file = workspace + "/" + RUNNING_GRAPH_FILE_NAME; + try { + std::ofstream ofs(running_graph_file, + std::ofstream::out | std::ofstream::trunc); + ofs << name; + ofs.close(); + LOG(INFO) << "Successfully set running graph: " << name; + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to set running graph: " << name + << ", error: " << e.what(); + } +} + +std::string WorkDirManipulator::GetRunningGraph() { + auto running_graph_file = workspace + "/" + RUNNING_GRAPH_FILE_NAME; + std::ifstream ifs(running_graph_file); + if (!ifs.is_open()) { + LOG(ERROR) << "Fail to open running graph file: " << running_graph_file; + return ""; + } + std::string line; + std::getline(ifs, line); + return line; +} + +// GraphName can be specified in the config file or in the argument. +gs::Result WorkDirManipulator::CreateGraph( + const YAML::Node& yaml_config) { + // First check graph exits + if (!yaml_config["name"]) { + return gs::Result( + gs::Status(gs::StatusCode::InvalidSchema, + "Graph name is not specified"), + seastar::sstring("Graph name is not specified")); + } + auto graph_name = yaml_config["name"].as(); + + if (is_graph_exist(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::AlreadyExists, "Graph already exists"), + seastar::sstring("graph " + graph_name + " already exists")); + } + + // First check whether yaml is valid + auto schema_result = gs::Schema::LoadFromYamlNode(yaml_config); + if (!schema_result.ok()) { + return gs::Result( + seastar::sstring(schema_result.status().error_message())); + } + auto& schema = schema_result.value(); + // dump schema to file. + auto dump_res = dump_graph_schema(yaml_config, graph_name); + if (!dump_res.ok()) { + return gs::Result(gs::Status( + gs::StatusCode::PermissionError, + "Fail to dump graph schema: " + dump_res.status().error_message())); + } + VLOG(10) << "Successfully dump graph schema to file: " << graph_name << ", " + << GetGraphSchemaPath(graph_name); + + return gs::Result( + seastar::sstring("successfuly created graph ")); +} + +gs::Result WorkDirManipulator::GetGraphSchemaString( + const std::string& graph_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::NotExists, + "Graph not exists: " + graph_name), + seastar::sstring()); + } + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph schema file is expected, but not exists: " + schema_file)); + } + // read schema file and output to string + auto schema_str_res = gs::get_string_from_yaml(schema_file); + if (!schema_str_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::NotExists, + "Failt to read schema file: " + schema_file + + ", error: " + schema_str_res.status().error_message())); + } else { + return gs::Result(schema_str_res.value()); + } +} + +gs::Result WorkDirManipulator::GetGraphSchema( + const std::string& graph_name) { + LOG(INFO) << "Get graph schema: " << graph_name; + gs::Schema schema; + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph schema file is expected, but not exists: " + schema_file)); + } + // Load schema from schema_file + try { + LOG(INFO) << "Load graph schema from file: " << schema_file; + schema = gs::Schema::LoadFromYaml(schema_file); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to load graph schema: " << schema_file + << ", error: " << e.what(); + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + + ", for graph: " + graph_name + e.what())); + } + return gs::Result(schema); +} + +gs::Result WorkDirManipulator::GetDataDirectory( + const std::string& graph_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::NotExists, + "Graph not exists: " + graph_name), + seastar::sstring()); + } + auto data_dir = GetGraphIndicesDir(graph_name); + if (!std::filesystem::exists(data_dir)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph data directory is expected, but not exists: " + data_dir)); + } + return gs::Result(data_dir); +} + +gs::Result WorkDirManipulator::ListGraphs() { + // list all graph schema files under data_workspace + YAML::Node yaml_list; + auto data_workspace = workspace + "/" + DATA_DIR_NAME; + for (const auto& entry : + std::filesystem::directory_iterator(data_workspace)) { + if (entry.is_directory()) { + auto graph_name = entry.path().filename().string(); + // visit graph.yaml under data/graph_name/graph.yaml + auto graph_path = GetGraphSchemaPath(graph_name); + VLOG(10) << "Check graph path: " << graph_path; + if (!std::filesystem::exists(graph_path)) { + continue; + } + try { + auto graph_schema_str_res = YAML::LoadFile(graph_path); + yaml_list.push_back(graph_schema_str_res); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to parse graph schema file: " << graph_path + << ", error: " << e.what(); + } + } + } + auto json_str = gs::get_string_from_yaml(yaml_list); + if (!json_str.ok()) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to convert yaml to json: " + json_str.status().error_message())); + } + return gs::Result(json_str.value()); +} + +gs::Result WorkDirManipulator::DeleteGraph( + const std::string& graph_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::NotExists, + "Graph not exists: " + graph_name), + seastar::sstring("graph " + graph_name + " not exists")); + } + if (is_graph_running(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::IllegalOperation, + "Can not remove a running " + graph_name), + seastar::sstring("graph " + graph_name + + " is running, can not be removed")); + } + if (is_graph_locked(graph_name)) { + return gs::Result( + gs::Status(gs::StatusCode::IllegalOperation, + "Can not remove graph " + graph_name + + ", since data loading ongoing"), + seastar::sstring("Can not remove graph " + graph_name + + ", since data loading ongoing")); + } + // remove the graph directory + try { + auto graph_path = get_graph_dir(graph_name); + std::filesystem::remove_all(graph_path); + } catch (const std::exception& e) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to remove graph directory: " + graph_name), + seastar::sstring("Fail to remove graph directory: " + graph_name)); + } + return gs::Result( + gs::Status::OK(), "Successfully delete graph: " + graph_name); +} + +gs::Result WorkDirManipulator::LoadGraph( + const std::string& graph_name, const YAML::Node& yaml_node, + int32_t loading_thread_num) { + // First check whether graph exists + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + if (is_graph_locked(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::IllegalOperation, + "Graph is locked: " + graph_name + + ", either service is running on graph, or graph is loading")); + } + // Then check graph is already loaded + if (is_graph_loaded(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::IllegalOperation, + "Graph is already loaded, can not be loaded twice: " + graph_name)); + } + // check is graph locked + if (is_graph_running(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::IllegalOperation, + "Graph is already running, can not be loaded: " + graph_name)); + } + if (!try_lock_graph(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::IllegalOperation, "Fail to lock graph: " + graph_name)); + } + + // No need to check whether graph exists, because it is checked in LoadGraph + // First load schema + auto schema_file = GetGraphSchemaPath(graph_name); + gs::Schema schema; + try { + schema = gs::Schema::LoadFromYaml(schema_file); + } catch (const std::exception& e) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + + ", for graph: " + graph_name)); + } + VLOG(1) << "Loaded schema, vertex label num: " << schema.vertex_label_num() + << ", edge label num: " << schema.edge_label_num(); + + auto loading_config_res = + gs::LoadingConfig::ParseFromYamlNode(schema, yaml_node); + if (!loading_config_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + loading_config_res.status().error_message())); + } + // dump to file + auto loading_config = loading_config_res.value(); + std::string temp_file_name = graph_name + "_bulk_loading_config.yaml"; + auto temp_file_path = TMP_DIR + "/" + temp_file_name; + auto dump_res = dump_yaml_to_file(yaml_node, temp_file_path); + if (!dump_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump loading config to file: " + temp_file_path + + ", error: " + dump_res.status().error_message())); + } + + auto res = LoadGraph(temp_file_path, graph_name, loading_thread_num); + if (!res.ok()) { + return gs::Result(res.status()); + } + // unlock graph + unlock_graph(graph_name); + + return gs::Result(res.status(), res.value()); +} + +gs::Result WorkDirManipulator::GetProceduresByGraphName( + const std::string& graph_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + // get graph schema file, and get procedure lists. + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph schema file is expected, but not exists: " + schema_file)); + } + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(schema_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + ", error: " + e.what())); + } + if (schema_node["stored_procedures"]) { + auto procedure_node = schema_node["stored_procedures"]; + if (procedure_node["enable_lists"]) { + auto procedures = procedure_node["enable_lists"]; + if (procedures.IsSequence()) { + std::vector procedure_list; + for (const auto& procedure : procedures) { + procedure_list.push_back(procedure.as()); + } + LOG(INFO) << "Enabled procedures found: " << graph_name + << ", schema file: " << schema_file + << ", procedure list: " << gs::to_string(procedure_list); + return get_all_procedure_yamls(graph_name, procedure_list); + } + } + } + LOG(INFO) << "No enabled procedures found: " << graph_name + << ", schema file: " << schema_file; + return get_all_procedure_yamls( + graph_name); // should be all procedures, not enabled only. +} + +gs::Result +WorkDirManipulator::GetProcedureByGraphAndProcedureName( + const std::string& graph_name, const std::string& procedure_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + // get graph schema file, and get procedure lists. + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph schema file is expected, but not exists: " + schema_file)); + } + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(schema_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + ", error: " + e.what())); + } + // get yaml file in plugin directory. + auto plugin_dir = get_graph_plugin_dir(graph_name); + if (!std::filesystem::exists(plugin_dir)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph plugin directory is expected, but not exists: " + plugin_dir)); + } + auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; + if (!std::filesystem::exists(plugin_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + } + // check whether procedure is enabled. + YAML::Node plugin_node; + try { + plugin_node = YAML::LoadFile(plugin_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph plugin: " + plugin_file + ", error: " + e.what())); + } + plugin_node["enabled"] = false; + + if (schema_node["stored_procedures"]) { + auto procedure_node = schema_node["stored_procedures"]; + if (procedure_node["enable_lists"]) { + auto procedures = procedure_node["enable_lists"]; + if (procedures.IsSequence()) { + std::vector procedure_list; + for (const auto& procedure : procedures) { + procedure_list.push_back(procedure.as()); + } + if (std::find(procedure_list.begin(), procedure_list.end(), + procedure_name) != procedure_list.end()) { + // add enabled: true to the plugin yaml. + plugin_node["enabled"] = true; + } + } + } else { + LOG(INFO) << "No enabled procedures found: " << graph_name + << ", schema file: " << schema_file; + } + } + // yaml_list to string + YAML::Emitter emitter; + emitter << plugin_node; + auto str = emitter.c_str(); + return gs::Result(std::move(str)); +} + +seastar::future WorkDirManipulator::CreateProcedure( + const std::string& graph_name, const std::string& parameter) { + if (!is_graph_exist(graph_name)) { + return seastar::make_ready_future("Graph not exists: " + + graph_name); + } + // check procedure exits + auto plugin_dir = get_graph_plugin_dir(graph_name); + if (!std::filesystem::exists(plugin_dir)) { + try { + std::filesystem::create_directory(plugin_dir); + } catch (const std::exception& e) { + return seastar::make_ready_future( + "Fail to create plugin directory: " + plugin_dir); + } + } + // load parameter as json, and do some check + nlohmann::json json; + try { + json = nlohmann::json::parse(parameter); + } catch (const std::exception& e) { + return seastar::make_exception_future( + "Fail to parse parameter as json: " + parameter); + } + // check required fields is give. + auto res = create_procedure_sanity_check(json); + if (!res.ok()) { + return seastar::make_exception_future( + res.status().error_message()); + } + LOG(INFO) << "Pass sanity check for procedure: " + << json["name"].get(); + // get procedure name + auto procedure_name = json["name"].get(); + // check whether procedure already exists. + auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; + if (std::filesystem::exists(plugin_file)) { + return seastar::make_exception_future( + "Procedure already exists: " + procedure_name); + } + return generate_procedure(json).then_wrapped([json](auto&& fut) { + try { + auto res = fut.get(); + bool enable = true; // default enable. + if (json.contains("enable")) { + if (json["enable"].is_boolean()) { + enable = json["enable"].get(); + } else if (json["enable"].is_string()) { + auto enable_str = json["enable"].get(); + if (enable_str == "true" || enable_str == "True" || + enable_str == "TRUE") { + enable = true; + } else { + enable = false; + } + } else { + return seastar::make_ready_future( + "Fail to parse enable field: " + json["enable"].dump()); + } + } + LOG(INFO) << "Enable: " << std::to_string(enable); + + // If create procedure success, update graph schema (dump to file) + // and add to plugin list. this is critical, and should be + // transactional. + if (enable) { + LOG(INFO) + << "Procedure is enabled, add to graph schema and plugin list."; + return add_procedure_to_graph(json, res); + } else { + // Not enabled, do nothing. + LOG(INFO) << "Procedure is not enabled, do nothing."; + } + + return seastar::make_ready_future( + seastar::sstring("Successfully create procedure")); + } catch (const std::exception& e) { + return seastar::make_ready_future( + "Fail to generate procedure: " + std::string(e.what())); + } + return seastar::make_ready_future( + "Fail to generate procedure"); + }); +} + +gs::Result WorkDirManipulator::DeleteProcedure( + const std::string& graph_name, const std::string& procedure_name) { + LOG(INFO) << "Delete procedure: " << procedure_name + << " on graph: " << graph_name; + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + // delete from graph schema + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph schema file is expected, but not exists: " + schema_file)); + } + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(schema_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + ", error: " + e.what())); + } + + if (schema_node["stored_procedures"]) { + auto procedure_node = schema_node["stored_procedures"]; + if (procedure_node["enable_lists"]) { + auto procedures = procedure_node["enable_lists"]; + if (procedures.IsSequence()) { + std::vector procedure_list; + for (const auto& procedure : procedures) { + procedure_list.push_back(procedure.as()); + } + auto it = std::find(procedure_list.begin(), procedure_list.end(), + procedure_name); + if (it != procedure_list.end()) { + procedure_list.erase(it); + procedures = procedure_list; + VLOG(1) << "Successfully removed " << procedure_name + << " from procedure list" << gs::to_string(procedure_list); + procedure_node["enable_lists"] = procedures; + // dump to file. + auto dump_res = dump_yaml_to_file(schema_node, schema_file); + if (dump_res.ok()) { + LOG(INFO) << "Dump graph schema to file: " << schema_file; + } else { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to dump graph schema: " + schema_file + + ", error: " + dump_res.status().error_message())); + } + } + } else { + VLOG(10) << "No enabled procedures found: " << graph_name + << ", schema file: " << schema_file; + } + } else { + LOG(INFO) << "No enabled procedures found: " << graph_name + << ", schema file: " << schema_file; + } + } + // remove the plugin file and dynamic lib + auto plugin_dir = get_graph_plugin_dir(graph_name); + if (!std::filesystem::exists(plugin_dir)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph plugin directory is expected, but not exists: " + plugin_dir)); + } + auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; + if (!std::filesystem::exists(plugin_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + } + try { + std::filesystem::remove(plugin_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to remove plugin file: " + plugin_file + ", error: " + e.what())); + } + auto plugin_lib = plugin_dir + "/lib" + procedure_name + ".so"; + if (!std::filesystem::exists(plugin_lib)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "plugin lib not found " + plugin_lib)); + } + try { + std::filesystem::remove(plugin_lib); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to remove plugin lib: " + plugin_lib + ", error: " + e.what())); + } + return gs::Result(gs::Status::OK(), + "Successfully delete procedure"); +} + +// we only support update the description and enable status. +gs::Result WorkDirManipulator::UpdateProcedure( + const std::string& graph_name, const std::string& procedure_name, + const std::string& parameters) { + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + // check procedure exits. + auto plugin_dir = get_graph_plugin_dir(graph_name); + if (!std::filesystem::exists(plugin_dir)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph plugin directory is expected, but not exists: " + plugin_dir)); + } + auto plugin_file = plugin_dir + "/" + procedure_name + ".yaml"; + if (!std::filesystem::exists(plugin_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "plugin not found " + plugin_file)); + } + // load parameter as json, and do some check + nlohmann::json json; + try { + json = nlohmann::json::parse(parameters); + } catch (const std::exception& e) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to parse parameter as json: " + parameters)); + } + VLOG(1) << "Successfully parse json paramters: " << json.dump(); + // load plugin_file as yaml + YAML::Node plugin_node; + try { + plugin_node = YAML::LoadFile(plugin_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph plugin: " + plugin_file + ", error: " + e.what())); + } + // update description and enable status. + if (json.contains("description")) { + auto new_description = json["description"]; + VLOG(10) << "Update description: " + << new_description; // update description + plugin_node["description"] = new_description.get(); + } + + bool enabled; + if (json.contains("enable")) { + VLOG(1) << "Enable is specified in the parameter:" << json["enable"].dump(); + if (json["enable"].is_boolean()) { + enabled = json["enable"].get(); + } else if (json["enable"].is_string()) { + auto enable_str = json["enable"].get(); + if (enable_str == "true" || enable_str == "True" || + enable_str == "TRUE") { + enabled = true; + } else { + enabled = false; + } + } else { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to parse enable field: " + json["enable"].dump())); + } + plugin_node["enable"] = enabled; + } + + // dump to file. + auto dump_res = dump_yaml_to_file(plugin_node, plugin_file); + if (!dump_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump plugin yaml to file: " + plugin_file + + ", error: " + dump_res.status().error_message())); + } + VLOG(10) << "Dump plugin yaml to file: " << plugin_file; + // if enable is specified in the parameter, update graph schema file. + if (enabled) { + return enable_procedure_on_graph(graph_name, procedure_name); + } else { + return disable_procedure_on_graph(graph_name, procedure_name); + } +} + +gs::Result WorkDirManipulator::GetProcedureLibPath( + const std::string& graph_name, const std::string& procedure_name) { + if (!is_graph_exist(graph_name)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph not exists: " + graph_name)); + } + // get the plugin dir and append procedure_name + auto plugin_dir = get_graph_plugin_dir(graph_name); + if (!std::filesystem::exists(plugin_dir)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph plugin directory is expected, but not exists: " + plugin_dir)); + } + auto plugin_so_path = plugin_dir + "/lib" + procedure_name + ".so"; + if (!std::filesystem::exists(plugin_so_path)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, + "Graph plugin so file is expected, but not exists: " + plugin_so_path)); + } + return gs::Result(plugin_so_path); +} + +std::string WorkDirManipulator::GetGraphSchemaPath( + const std::string& graph_name) { + return get_graph_dir(graph_name) + "/" + GRAPH_SCHEMA_FILE_NAME; +} + +std::string WorkDirManipulator::get_graph_lock_file( + const std::string& graph_name) { + return get_graph_dir(graph_name) + "/" + LOCK_FILE; +} + +std::string WorkDirManipulator::GetGraphIndicesDir( + const std::string& graph_name) { + return get_graph_dir(graph_name) + "/" + GRAPH_INDICES_DIR_NAME; +} + +std::string WorkDirManipulator::get_graph_indices_file( + const std::string& graph_name) { + return get_graph_dir(graph_name) + GRAPH_INDICES_DIR_NAME + "/" + + GRAPH_INDICES_FILE_NAME; +} + +std::string WorkDirManipulator::get_graph_plugin_dir( + const std::string& graph_name) { + return get_graph_dir(graph_name) + "/" + GRAPH_PLUGIN_DIR_NAME; +} + +// graph_name can be a path, first try as it is absolute path, or +// relative path +std::string WorkDirManipulator::get_graph_dir(const std::string& graph_name) { + if (std::filesystem::exists(graph_name)) { + return graph_name; + } + return workspace + "/" + DATA_DIR_NAME + "/" + graph_name; +} + +bool WorkDirManipulator::is_graph_exist(const std::string& graph_name) { + auto graph_path = GetGraphSchemaPath(graph_name); + return std::filesystem::exists(graph_path); +} + +bool WorkDirManipulator::is_graph_loaded(const std::string& graph_name) { + return std::filesystem::exists(get_graph_indices_file(graph_name)); +} + +bool WorkDirManipulator::is_graph_running(const std::string& graph_name) { + return GetRunningGraph() == graph_name; +} + +bool WorkDirManipulator::is_graph_locked(const std::string& graph_name) { + auto lock_file = get_graph_lock_file(graph_name); + return std::filesystem::exists(lock_file); +} + +bool WorkDirManipulator::try_lock_graph(const std::string& graph_name) { + auto lock_file = get_graph_lock_file(graph_name); + if (std::filesystem::exists(lock_file)) { + return false; + } + std::ofstream fout(lock_file); + if (!fout.is_open()) { + return false; + } + fout.close(); + return true; +} + +void WorkDirManipulator::unlock_graph(const std::string& graph_name) { + auto lock_file = get_graph_lock_file(graph_name); + if (std::filesystem::exists(lock_file)) { + std::filesystem::remove(lock_file); + } +} + +std::string WorkDirManipulator::get_engine_config_path() { + return workspace + "/conf/" + CONF_ENGINE_CONFIG_FILE_NAME; +} + +bool WorkDirManipulator::ensure_graph_dir_exists( + const std::string& graph_name) { + auto graph_path = get_graph_dir(graph_name); + if (!std::filesystem::exists(graph_path)) { + std::filesystem::create_directory(graph_path); + } + return std::filesystem::exists(graph_path); +} + +gs::Result WorkDirManipulator::dump_graph_schema( + const YAML::Node& yaml_config, const std::string& graph_name) { + if (!ensure_graph_dir_exists(graph_name)) { + return {gs::Status(gs::StatusCode::PermissionError, + "Fail to create graph directory")}; + } + auto graph_path = GetGraphSchemaPath(graph_name); + VLOG(10) << "Dump graph schema to file: " << graph_path; + std::ofstream fout(graph_path); + if (!fout.is_open()) { + return {gs::Status(gs::StatusCode::PermissionError, "Fail to open file")}; + } + fout << yaml_config; + fout.close(); + VLOG(10) << "Successfully dump graph schema to file: " << graph_path; + return gs::Result(gs::Status::OK()); +} + +gs::Result WorkDirManipulator::LoadGraph( + const std::string& config_file_path, const std::string& graph_name, + int32_t loading_thread_num) { + // TODO: call graph_loader. + auto schema_file = GetGraphSchemaPath(graph_name); + auto cur_indices_dir = GetGraphIndicesDir(graph_name); + // system call to graph_loader schema_file, loading_config, cur_indices_dir + std::string cmd_string = "graph_loader " + schema_file + " " + + config_file_path + " " + cur_indices_dir + " " + + std::to_string(loading_thread_num); + LOG(INFO) << "Call graph_loader: " << cmd_string; + auto res = std::system(cmd_string.c_str()); + if (res != 0) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to load graph: " + graph_name + + ", error code: " + std::to_string(res))); + } + + return gs::Result( + gs::Status::OK(), "Successfully load data to graph: " + graph_name); +} + +gs::Result WorkDirManipulator::create_procedure_sanity_check( + const nlohmann::json& json) { + // check required fields is give. + CHECK_JSON_FIELD(json, "bound_graph"); + CHECK_JSON_FIELD(json, "description"); + CHECK_JSON_FIELD(json, "enable"); + CHECK_JSON_FIELD(json, "name"); + CHECK_JSON_FIELD(json, "query"); + CHECK_JSON_FIELD(json, "type"); + auto type = json["type"].get(); + if (type == "cypher" || type == "CYPHER") { + LOG(INFO) << "Cypher procedure, name: " << json["name"].get() + << ", enable: " << json["enable"].get(); + } else if (type == "CPP" || type == "cpp") { + CHECK_JSON_FIELD(json, "params"); + CHECK_JSON_FIELD(json, "returns"); + LOG(INFO) << "Native procedure, name: " << json["name"].get() + << ", enable: " << json["enable"].get(); + } else { + return gs::Result( + gs::Status(gs::StatusCode::InValidArgument, + "Procedure type is not supported: " + type)); + } + + return gs::Result(gs::Status::OK()); +} + +seastar::future WorkDirManipulator::generate_procedure( + const nlohmann::json& json) { + LOG(INFO) << "Generate procedure: " << json.dump(); + auto codegen_bin = gs::find_codegen_bin(); + auto work_directory = std::string(server::CodegenProxy::DEFAULT_CODEGEN_DIR); + // dump json["query"] to file. + auto query = json["query"].get(); + auto name = json["name"].get(); + auto type = json["type"].get(); + auto bounded_graph = json["bound_graph"].get(); + std::string query_file; + if (type == "cypher" || type == "CYPHER") { + query_file = work_directory + "/" + name + ".cypher"; + } else if (type == "CPP" || type == "cpp") { + query_file = work_directory + "/" + name + ".cpp"; + } else { + return seastar::make_exception_future( + "Procedure type is not supported: " + type); + } + // dump query string as text to query_file + try { + std::ofstream fout(query_file); + if (!fout.is_open()) { + return seastar::make_exception_future( + "Fail to open query file: " + query_file); + } + fout << query; + fout.close(); + } catch (const std::exception& e) { + return seastar::make_exception_future( + "Fail to dump query to file: " + query_file + + ", error: " + std::string(e.what())); + } + + if (!is_graph_exist(bounded_graph)) { + return seastar::make_exception_future( + "Graph not exists: " + bounded_graph); + } + auto output_dir = get_graph_plugin_dir(bounded_graph); + if (!std::filesystem::exists(output_dir)) { + std::filesystem::create_directory(output_dir); + } + auto schema_path = GetGraphSchemaPath(bounded_graph); + auto engine_config = get_engine_config_path(); + + return CodegenProxy::CallCodegenCmd(query_file, name, work_directory, + output_dir, schema_path, engine_config, + codegen_bin) + .then_wrapped([name, output_dir](auto&& f) { + try { + auto res = f.get(); + std::string so_file; + { + std::stringstream ss; + ss << output_dir << "/lib" << name << ".so"; + so_file = ss.str(); + } + LOG(INFO) << "Check so file: " << so_file; + + if (!std::filesystem::exists(so_file)) { + return seastar::make_exception_future( + "Fail to generate procedure, so file not exists: " + so_file); + } + std::string yaml_file; + { + std::stringstream ss; + ss << output_dir << "/" << name << ".yaml"; + yaml_file = ss.str(); + } + LOG(INFO) << "Check yaml file: " << yaml_file; + if (!std::filesystem::exists(yaml_file)) { + return seastar::make_exception_future( + "Fail to generate procedure, yaml file not exists: " + + yaml_file); + } + return seastar::make_ready_future( + seastar::sstring{yaml_file}); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to generate procedure, error: " << e.what(); + return seastar::make_exception_future( + "Fail to generate procedure, error: " + std::string(e.what())); + } catch (...) { + LOG(ERROR) << "Fail to generate procedure, unknown error"; + return seastar::make_exception_future( + "Fail to generate procedure, unknown error"); + } + }); +} + +seastar::future WorkDirManipulator::add_procedure_to_graph( + const nlohmann::json& json, const std::string& proc_yaml_config_file) { + try { + YAML::Node proc_config_node; + proc_config_node = YAML::LoadFile(proc_yaml_config_file); + } catch (const std::exception& e) { + return seastar::make_exception_future(gs::Status( + gs::StatusCode::InternalError, + "Fail to load procedure config file: " + proc_yaml_config_file + + ", error: " + e.what())); + } + // get graph_name from json + auto graph_name = json["bound_graph"].get(); + auto proc_name = json["name"].get(); + if (proc_name.empty()) { + return seastar::make_exception_future( + "Procedure name is empty, can not add to graph: " + graph_name); + } + // get graph schema file + auto graph_schema_file = GetGraphSchemaPath(graph_name); + // load graph schema + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(graph_schema_file); + } catch (const std::exception& e) { + return seastar::make_exception_future( + "Fail to load graph schema: " + graph_schema_file + + ", error: " + e.what()); + } + // get plugin list + if (!schema_node) { + return seastar::make_exception_future( + "Graph schema is empty, can not add procedure to graph: " + graph_name); + } + if (!schema_node["stored_procedures"]) { + schema_node["stored_procedures"] = YAML::Node(YAML::NodeType::Map); + } + auto stored_procedures = schema_node["stored_procedures"]; + if (!stored_procedures["enable_lists"]) { + stored_procedures["enable_lists"] = YAML::Node(YAML::NodeType::Sequence); + } + auto enable_lists = stored_procedures["enable_lists"]; + // check whether procedure is already in the list, if so, then we raise + // error: procedure already exists. + for (const auto& item : enable_lists) { + if (item.as() == proc_name) { + return seastar::make_exception_future( + "Procedure already exists in graph: " + graph_name); + } + } + enable_lists.push_back(proc_name); + // dump schema to file + try { + std::ofstream fout(graph_schema_file); + if (!fout.is_open()) { + return seastar::make_exception_future( + "Fail to open graph schema file: " + graph_schema_file); + } + fout << schema_node; + fout.close(); + } catch (const std::exception& e) { + return seastar::make_exception_future( + "Fail to dump graph schema to file: " + graph_schema_file + + ", error: " + e.what()); + } + return seastar::make_ready_future( + seastar::sstring("Successfully create procedure")); +} + +gs::Result WorkDirManipulator::get_all_procedure_yamls( + const std::string& graph_name, + const std::vector& procedure_names) { + YAML::Node yaml_list; + auto plugin_dir = get_graph_plugin_dir(graph_name); + // iterate all .yamls in plugin_dir + if (std::filesystem::exists(plugin_dir)) { + for (const auto& entry : std::filesystem::directory_iterator(plugin_dir)) { + if (entry.path().extension() == ".yaml") { + auto procedure_yaml_file = entry.path().string(); + try { + auto procedure_yaml_node = YAML::LoadFile(procedure_yaml_file); + procedure_yaml_node["enabled"] = false; + if (!procedure_yaml_node["name"]) { + LOG(ERROR) << "Procedure yaml file not contains name: " + << procedure_yaml_file; + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Procedure yaml file not contains name: " + + procedure_yaml_file)); + } + auto proc_name = procedure_yaml_node["name"].as(); + if (std::find(procedure_names.begin(), procedure_names.end(), + proc_name) != procedure_names.end()) { + // only add the procedure yaml file that is in procedure_names. + procedure_yaml_node["enabled"] = true; + } + yaml_list.push_back(procedure_yaml_node); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to load procedure yaml file: " + << procedure_yaml_file << ", error: " << e.what(); + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load procedure yaml file: " + procedure_yaml_file + + ", error: " + e.what())); + } + } + } + } + // dump to json + auto res = gs::get_string_from_yaml(yaml_list); + if (!res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump procedure yaml list to json, error: " + + res.status().error_message())); + } + return gs::Result(std::move(res.value())); +} + +// get all procedures for graph, all set to disabled. +gs::Result WorkDirManipulator::get_all_procedure_yamls( + const std::string& graph_name) { + YAML::Node yaml_list; + auto plugin_dir = get_graph_plugin_dir(graph_name); + // iterate all .yamls in plugin_dir + if (std::filesystem::exists(plugin_dir)) { + for (const auto& entry : std::filesystem::directory_iterator(plugin_dir)) { + if (entry.path().extension() == ".yaml") { + auto procedure_yaml_file = entry.path().string(); + try { + auto procedure_yaml_node = YAML::LoadFile(procedure_yaml_file); + procedure_yaml_node["enabled"] = false; + yaml_list.push_back(procedure_yaml_node); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to load procedure yaml file: " + << procedure_yaml_file << ", error: " << e.what(); + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load procedure yaml file: " + procedure_yaml_file + + ", error: " + e.what())); + } + } + } + } + // dump to json + auto res = gs::get_string_from_yaml(yaml_list); + if (!res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump procedure yaml list to json, error: " + + res.status().error_message())); + } + return gs::Result(std::move(res.value())); +} + +gs::Result WorkDirManipulator::get_procedure_yaml( + const std::string& graph_name, const std::string& procedure_name) { + auto procedure_yaml_file = + get_graph_plugin_dir(graph_name) + "/" + procedure_name + ".yaml"; + if (!std::filesystem::exists(procedure_yaml_file)) { + LOG(ERROR) << "Procedure yaml file not exists: " << procedure_yaml_file; + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Procedure yaml file not exists: " + procedure_yaml_file)); + } + try { + auto procedure_yaml_node = YAML::LoadFile(procedure_yaml_file); + // dump to json + YAML::Emitter emitter; + emitter << procedure_yaml_node; + auto str = emitter.c_str(); + return gs::Result(std::move(str)); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to load procedure yaml file: " << procedure_yaml_file + << ", error: " << e.what(); + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to load procedure yaml file: " + procedure_yaml_file + + ", error: " + e.what())); + } + return gs::Result( + gs::Status(gs::StatusCode::InternalError, "Unknown error")); +} + +gs::Result WorkDirManipulator::enable_procedure_on_graph( + const std::string& graph_name, const std::string& procedure_name) { + LOG(INFO) << "Enabling procedure " << procedure_name << " on graph " + << graph_name; + + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph schema file not exists: " + + schema_file + ", graph: " + graph_name)); + } + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(schema_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + ", error: " + e.what())); + } + if (!schema_node["stored_procedures"]) { + schema_node["stored_procedures"] = YAML::Node(YAML::NodeType::Map); + } + auto stored_procedures = schema_node["stored_procedures"]; + if (!stored_procedures["enable_lists"]) { + stored_procedures["enable_lists"] = YAML::Node(YAML::NodeType::Sequence); + } + auto enable_lists = stored_procedures["enable_lists"]; + // check whether procedure is already in the list, if so, then we raise + // error: procedure already exists. + for (const auto& item : enable_lists) { + if (item.as() == procedure_name) { + return gs::Result( + gs::Status(gs::StatusCode::OK, + "Procedure already exists in graph: " + graph_name)); + } + } + enable_lists.push_back(procedure_name); + // dump schema to file + auto dump_res = dump_yaml_to_file(schema_node, schema_file); + if (!dump_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump graph schema: " + schema_file + + ", error: " + dump_res.status().error_message())); + } + return gs::Result(gs::Status::OK(), "Success"); +} + +gs::Result WorkDirManipulator::disable_procedure_on_graph( + const std::string& graph_name, const std::string& procedure_name) { + LOG(INFO) << "Disabling procedure " << procedure_name << " on graph " + << graph_name; + + auto schema_file = GetGraphSchemaPath(graph_name); + if (!std::filesystem::exists(schema_file)) { + return gs::Result(gs::Status( + gs::StatusCode::NotExists, "Graph schema file not exists: " + + schema_file + ", graph: " + graph_name)); + } + YAML::Node schema_node; + try { + schema_node = YAML::LoadFile(schema_file); + } catch (const std::exception& e) { + return gs::Result(gs::Status( + gs::StatusCode::InternalError, + "Fail to load graph schema: " + schema_file + ", error: " + e.what())); + } + if (!schema_node["stored_procedures"]) { + schema_node["stored_procedures"] = YAML::Node(YAML::NodeType::Map); + } + auto stored_procedures = schema_node["stored_procedures"]; + if (!stored_procedures["enable_lists"]) { + stored_procedures["enable_lists"] = YAML::Node(YAML::NodeType::Sequence); + } + auto enable_lists = stored_procedures["enable_lists"]; + // check whether procedure is already in the list, if so, then we raise + // error: procedure already exists. + // remove procedure from enable_lists + auto new_enable_list = YAML::Node(YAML::NodeType::Sequence); + int ind = -1; + for (auto iter = enable_lists.begin(); iter != enable_lists.end(); iter++) { + ind += 1; + if (iter->as() == procedure_name) { + LOG(INFO) << "Found procedure " << procedure_name << " in enable_lists"; + break; + } else { + new_enable_list.push_back(*iter); + } + } + + LOG(INFO) << "after remove: " << enable_lists; + stored_procedures["enable_lists"] = new_enable_list; + schema_node["stored_procedures"] = stored_procedures; + // dump schema to file + auto dump_res = dump_yaml_to_file(schema_node, schema_file); + if (!dump_res.ok()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump graph schema: " + schema_file + + ", error: " + dump_res.status().error_message())); + } + return gs::Result(gs::Status::OK(), "Success"); +} + +gs::Result WorkDirManipulator::dump_yaml_to_file( + const YAML::Node& yaml_node, const std::string& procedure_yaml_file) { + try { + YAML::Emitter emitter; + emitter << yaml_node; + auto str = emitter.c_str(); + std::ofstream fout(procedure_yaml_file); + if (!fout.is_open()) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to open file: " + procedure_yaml_file + + ", error: " + std::string(std::strerror(errno)))); + } + fout << str; + fout.close(); + } catch (const std::exception& e) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump yaml to file: " + procedure_yaml_file + + ", error: " + std::string(e.what()))); + } catch (...) { + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to dump yaml to file: " + procedure_yaml_file + + ", unknown error")); + } + LOG(INFO) << "Successfully dump yaml to file: " << procedure_yaml_file; + return gs::Result(gs::Status::OK(), "Success"); +} + +// define LOCK_FILE +// define DATA_DIR_NAME +const std::string WorkDirManipulator::LOCK_FILE = ".lock"; +const std::string WorkDirManipulator::DATA_DIR_NAME = "data"; +const std::string WorkDirManipulator::GRAPH_SCHEMA_FILE_NAME = "graph.yaml"; +const std::string WorkDirManipulator::GRAPH_INDICES_FILE_NAME = + "init_snapshot.bin"; +const std::string WorkDirManipulator::GRAPH_INDICES_DIR_NAME = "indices"; +const std::string WorkDirManipulator::GRAPH_PLUGIN_DIR_NAME = "plugins"; +const std::string WorkDirManipulator::CONF_ENGINE_CONFIG_FILE_NAME = + "engine_config.yaml"; +const std::string WorkDirManipulator::RUNNING_GRAPH_FILE_NAME = "RUNNING"; +const std::string WorkDirManipulator::TMP_DIR = "/tmp"; + +} // namespace server \ No newline at end of file diff --git a/flex/engines/http_server/workdir_manipulator.h b/flex/engines/http_server/workdir_manipulator.h new file mode 100644 index 000000000000..28ffb1534b13 --- /dev/null +++ b/flex/engines/http_server/workdir_manipulator.h @@ -0,0 +1,209 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_HTTP_SERVER_WORKDIR_MANIPULATOR_H_ +#define ENGINES_HTTP_SERVER_WORKDIR_MANIPULATOR_H_ + +#include +#include +#include +#include +#include +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/http_server/types.h" +#include "flex/storages/rt_mutable_graph/loading_config.h" +#include "flex/storages/rt_mutable_graph/schema.h" +#include "flex/utils/result.h" +#include "flex/utils/service_utils.h" +#include "flex/utils/yaml_utils.h" + +#include +#include "nlohmann/json.hpp" + +namespace server { + +/** + * @brief The class to manipulate the workspace. All methods are static. + */ +class WorkDirManipulator { + private: + // A static variable to store the workspace path, and shall not be changed + static std::string workspace; + + public: + static const std::string LOCK_FILE; + static const std::string DATA_DIR_NAME; + static const std::string GRAPH_SCHEMA_FILE_NAME; + static const std::string GRAPH_INDICES_FILE_NAME; + static const std::string GRAPH_INDICES_DIR_NAME; + static const std::string GRAPH_PLUGIN_DIR_NAME; + static const std::string CONF_ENGINE_CONFIG_FILE_NAME; + static const std::string RUNNING_GRAPH_FILE_NAME; + static const std::string TMP_DIR; + + static void SetWorkspace(const std::string& workspace_path); + + static void SetRunningGraph(const std::string& graph_name); + + static std::string GetRunningGraph(); + + /** + * @brief Create a graph with a given name and config. + * @param boost_ptree The config of the graph. + */ + static gs::Result CreateGraph(const YAML::Node& yaml_node); + + /** + * @brief Get a graph with a given name. + * @param graph_name The name of the graph. + */ + static gs::Result GetGraphSchemaString( + const std::string& graph_name); + + static gs::Result GetGraphSchema(const std::string& graph_name); + + static gs::Result GetDataDirectory( + const std::string& graph_name); + + /** + * @brief List all graphs. + * @return A vector of graph configs. + */ + static gs::Result ListGraphs(); + + /** + * @brief Delete a graph with a given name. + * @param graph_name The name of the graph. + */ + static gs::Result DeleteGraph( + const std::string& graph_name); + + /** + * @brief Load a graph with a given name and config. + * @param graph_name The name of the graph. + * @param yaml_node The config of the graph. + * @param loading_thread_num The number of threads to load the graph. + */ + static gs::Result LoadGraph(const std::string& graph_name, + const YAML::Node& yaml_node, + int32_t loading_thread_num); + + /** + * @brief Load a graph with a given name and config. + * @param yaml_config_file The config file of the graph. + * @param yaml_node The config of the graph. + * @param loading_thread_num The number of threads to load the graph. + */ + static gs::Result LoadGraph(const std::string& yaml_config_file, + const std::string& graph_name, + int32_t thread_num); + + /** + * @brief Get all procedures bound to the graph. + * @param graph_name The name of the graph. + * @param boost_ptree The config of the graph. + */ + static gs::Result GetProceduresByGraphName( + const std::string& graph_name); + + /** + * @brief Get a procedure with a given name. + * @param graph_name The name of the graph. + * @param procedure_name The name of the procedure. + */ + static gs::Result GetProcedureByGraphAndProcedureName( + const std::string& graph_name, const std::string& procedure_name); + + static seastar::future CreateProcedure( + const std::string& graph_name, const std::string& parameter); + + static gs::Result DeleteProcedure( + const std::string& graph_name, const std::string& procedure_name); + + static gs::Result UpdateProcedure( + const std::string& graph_name, const std::string& procedure_name, + const std::string& parameter); + + static gs::Result GetProcedureLibPath( + const std::string& graph_name, const std::string& procedure_name); + + static std::string GetGraphSchemaPath(const std::string& graph_name); + + static std::string GetGraphIndicesDir(const std::string& graph_name); + + private: + static gs::Result create_procedure_sanity_check( + const nlohmann::json& json); + + static std::string get_graph_indices_file(const std::string& graph_name); + + static std::string get_graph_lock_file(const std::string& graph_name); + + static std::string get_graph_plugin_dir(const std::string& graph_name); + + static std::string get_graph_dir(const std::string& graph_name); + + static std::string get_engine_config_path(); + + static bool is_graph_exist(const std::string& graph_name); + + static bool is_graph_loaded(const std::string& graph_name); + + static bool is_graph_running(const std::string& graph_name); + + static bool is_graph_locked(const std::string& graph_name); + + static bool try_lock_graph(const std::string& graph_name); + + static void unlock_graph(const std::string& graph_name); + + static bool ensure_graph_dir_exists(const std::string& graph_name); + + static gs::Result dump_graph_schema( + const YAML::Node& yaml_config, const std::string& graph_name); + + // Generate the procedure, return the generated yaml config. + static seastar::future generate_procedure( + const nlohmann::json& json); + + static seastar::future add_procedure_to_graph( + const nlohmann::json& json, const std::string& proc_yaml_config); + + // Get all the procedure yaml configs in plugins directory, add additional + // enabled:false to each config. + static gs::Result get_all_procedure_yamls( + const std::string& graph_name); + + // Get all the procedure yaml configs in plugins directory, add additional + // enabled:true to enabled_list, add additional enabled:false to others. + static gs::Result get_all_procedure_yamls( + const std::string& graph_name, + const std::vector& enabled_list); + + static gs::Result get_procedure_yaml( + const std::string& graph_name, const std::string& procedure_names); + + static gs::Result enable_procedure_on_graph( + const std::string& graph_name, const std::string& procedure_name); + + static gs::Result disable_procedure_on_graph( + const std::string& graph_name, const std::string& procedure_name); + + static gs::Result dump_yaml_to_file( + const YAML::Node& node, const std::string& file_path); +}; +} // namespace server + +#endif // ENGINES_HTTP_SERVER_WORKDIR_MANIPULATOR_H_ \ No newline at end of file diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive index b5a47944b460..67d7c817f584 100755 --- a/flex/interactive/bin/gs_interactive +++ b/flex/interactive/bin/gs_interactive @@ -187,7 +187,7 @@ HOST_DB_ENV_FILE="${HOST_DB_HOME}/.env" DOCKER_DB_GRAPHSCOPE_HOME="/opt/flex" DOCKER_DB_GIE_HOME="${DOCKER_DB_GRAPHSCOPE_HOME}" -DOCKER_DB_SERVER_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/sync_server" +DOCKER_DB_SERVER_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/interactive_server" DOCKER_DB_GRAPH_IMPORT_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/graph_loader" DOCKER_DB_COMPILER_BIN="com.alibaba.graphscope.GraphServer" DOCKER_DB_GEN_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/load_plan_and_gen.sh" @@ -213,6 +213,7 @@ DATABASE_COMPILER_QUERY_TIMEOUT="20000" ## hiactor related default configuration DATABASE_COMPUTE_ENGINE_PORT="10000" +DATABASE_ADMIN_PORT="7777" DATABASE_COMPUTE_ENGINE_SHARD_NUM=1 ## directories @@ -354,6 +355,7 @@ function update_init_config_from_yaml(){ # append compiler port and engine port to DATABASE_PORTS DATABASE_PORTS="${DATABASE_COMPILER_BOLT_PORT}:${DATABASE_COMPILER_BOLT_PORT}" DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_COMPUTE_ENGINE_PORT}:${DATABASE_COMPUTE_ENGINE_PORT}" + DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_ADMIN_PORT}:${DATABASE_ADMIN_PORT}" } function update_engine_config_from_yaml(){ @@ -676,8 +678,9 @@ function do_gen_conf(){ #compute_engine echo "compute_engine:" >> ${output_config_file} echo " type: hiactor" >> ${output_config_file} - echo " hosts:" >> ${output_config_file} + echo " workers:" >> ${output_config_file} echo " - localhost:${DATABASE_COMPUTE_ENGINE_PORT}" >> ${output_config_file} + echo " admin_endpoint:${DATABASE_ADMIN_PORT} " >> ${output_config_file} echo " shard_num: ${DATABASE_COMPUTE_ENGINE_SHARD_NUM}" >> ${output_config_file} @@ -1169,7 +1172,7 @@ function do_destroy() { if ! check_process_not_running_in_container ${GIE_DB_CONTAINER_NAME} "GraphServer" "Service is running, please stop it first: ./bin/gs_interacitve service stop"; then exit 1 fi - if ! check_process_not_running_in_container ${GIE_DB_CONTAINER_NAME} "sync_server" "Service is running, please stop it first: ./bin/gs_interacitve service stop"; then + if ! check_process_not_running_in_container ${GIE_DB_CONTAINER_NAME} "interactive_server" "Service is running, please stop it first: ./bin/gs_interacitve service stop"; then exit 1 fi diff --git a/flex/interactive/docker/interactive-base.Dockerfile b/flex/interactive/docker/interactive-base.Dockerfile index 4d9a9e5ae38b..53baac903723 100644 --- a/flex/interactive/docker/interactive-base.Dockerfile +++ b/flex/interactive/docker/interactive-base.Dockerfile @@ -23,7 +23,7 @@ ENV TZ=Asia/Shanghai RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update && apt-get install -y protobuf-compiler libprotobuf-dev maven git vim curl \ - wget python3 make libc-ares-dev doxygen python3-pip net-tools curl default-jdk \ + wget python3 make libc-ares-dev doxygen python3-pip net-tools curl default-jdk nlohmann-json3-dev \ libgoogle-glog-dev libopenmpi-dev libboost-all-dev libyaml-cpp-dev libprotobuf-dev libcrypto++-dev openssl # install libgrape-lite diff --git a/flex/interactive/examples/modern_graph/bulk_load.yaml b/flex/interactive/examples/modern_graph/bulk_load.yaml index 739302554d6e..6bda64715771 100644 --- a/flex/interactive/examples/modern_graph/bulk_load.yaml +++ b/flex/interactive/examples/modern_graph/bulk_load.yaml @@ -1,4 +1,4 @@ -graph: modern +graph: modern_graph loading_config: data_source: scheme: file # file, oss, s3, hdfs; only file is supported now diff --git a/flex/interactive/examples/modern_graph/modern_graph.yaml b/flex/interactive/examples/modern_graph/modern_graph.yaml index 41617bde53ab..d6b65c0fc544 100644 --- a/flex/interactive/examples/modern_graph/modern_graph.yaml +++ b/flex/interactive/examples/modern_graph/modern_graph.yaml @@ -1,4 +1,4 @@ -name: modern # then must have a modern dir under ${data} directory +name: modern_graph # then must have a modern dir under ${data} directory store_type: mutable_csr # v6d, groot, gart schema: vertex_types: @@ -30,7 +30,6 @@ schema: property_name: id property_type: primitive_type: DT_SIGNED_INT64 - x_csr_params: - property_id: 1 property_name: name property_type: diff --git a/flex/storages/rt_mutable_graph/loading_config.cc b/flex/storages/rt_mutable_graph/loading_config.cc index f97f633f3403..37b7b0dd2302 100644 --- a/flex/storages/rt_mutable_graph/loading_config.cc +++ b/flex/storages/rt_mutable_graph/loading_config.cc @@ -202,6 +202,7 @@ static bool parse_vertex_files( std::string file_path = files_node[i].as(); if (!access_file(data_location, file_path)) { LOG(ERROR) << "vertex file - " << file_path << " file not found..."; + return false; } std::filesystem::path path(file_path); files[label_id].emplace_back(std::filesystem::canonical(path)); @@ -404,6 +405,12 @@ static bool parse_bulk_load_config_file(const std::string& config_file, const Schema& schema, LoadingConfig& load_config) { YAML::Node root = YAML::LoadFile(config_file); + return parse_bulk_load_config_yaml(root, schema, load_config); +} + +static bool parse_bulk_load_config_yaml(const YAML::Node& root, + const Schema& schema, + LoadingConfig& load_config) { std::string data_location; load_config.scheme_ = "file"; // default data source is file load_config.method_ = "init"; @@ -516,8 +523,8 @@ static bool parse_bulk_load_config_file(const std::string& config_file, } } // namespace config_parsing -LoadingConfig LoadingConfig::ParseFromYaml(const Schema& schema, - const std::string& yaml_file) { +LoadingConfig LoadingConfig::ParseFromYamlFile(const Schema& schema, + const std::string& yaml_file) { LoadingConfig load_config(schema); if (!yaml_file.empty() && std::filesystem::exists(yaml_file)) { if (!config_parsing::parse_bulk_load_config_file(yaml_file, schema, @@ -528,6 +535,25 @@ LoadingConfig LoadingConfig::ParseFromYaml(const Schema& schema, return load_config; } +Result LoadingConfig::ParseFromYamlNode( + const Schema& schema, const YAML::Node& yaml_node) { + LoadingConfig load_config(schema); + try { + if (!yaml_node.IsNull()) { + if (!config_parsing::parse_bulk_load_config_yaml(yaml_node, schema, + load_config)) { + LOG(FATAL) << "Failed to parse bulk load config: "; + } + } + } catch (const YAML::Exception& e) { + return gs::Result( + gs::Status(gs::StatusCode::InvalidImportFile, + "Failed to parse yaml node: " + std::string(e.what())), + load_config); + } + return load_config; +} + LoadingConfig::LoadingConfig(const Schema& schema) : schema_(schema), scheme_("file"), method_("init"), format_("csv") {} diff --git a/flex/storages/rt_mutable_graph/loading_config.h b/flex/storages/rt_mutable_graph/loading_config.h index 15f2544fae2b..f596da59ae4f 100644 --- a/flex/storages/rt_mutable_graph/loading_config.h +++ b/flex/storages/rt_mutable_graph/loading_config.h @@ -63,7 +63,11 @@ namespace config_parsing { static bool parse_bulk_load_config_file(const std::string& config_file, const Schema& schema, LoadingConfig& load_config); -} + +static bool parse_bulk_load_config_yaml(const YAML::Node& yaml_node, + const Schema& schema, + LoadingConfig& load_config); +} // namespace config_parsing // Provide meta info about bulk loading. class LoadingConfig { @@ -74,8 +78,10 @@ class LoadingConfig { schema_label_type>; // src_label_t, dst_label_t, edge_label_t // Check whether loading config file is consistent with schema - static LoadingConfig ParseFromYaml(const Schema& schema, - const std::string& yaml_file); + static LoadingConfig ParseFromYamlFile(const Schema& schema, + const std::string& yaml_file); + static gs::Result ParseFromYamlNode( + const Schema& schema, const YAML::Node& yaml_node); LoadingConfig(const Schema& schema); @@ -171,6 +177,9 @@ class LoadingConfig { friend bool config_parsing::parse_bulk_load_config_file( const std::string& config_file, const Schema& schema, LoadingConfig& load_config); + + friend bool config_parsing::parse_bulk_load_config_yaml( + const YAML::Node& root, const Schema& schema, LoadingConfig& load_config); }; } // namespace gs diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc index 3168b2f3d558..9e059d928e19 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc @@ -35,6 +35,26 @@ MutablePropertyFragment::~MutablePropertyFragment() { } } +void MutablePropertyFragment::Clear() { + for (auto ptr : ie_) { + if (ptr != NULL) { + delete ptr; + } + } + for (auto ptr : oe_) { + if (ptr != NULL) { + delete ptr; + } + } + lf_indexers_.clear(); + vertex_data_.clear(); + ie_.clear(); + oe_.clear(); + vertex_label_num_ = 0; + edge_label_num_ = 0; + schema_.Clear(); +} + void MutablePropertyFragment::IngestEdge(label_t src_label, vid_t src_lid, label_t dst_label, vid_t dst_lid, label_t edge_label, timestamp_t ts, @@ -48,6 +68,8 @@ void MutablePropertyFragment::IngestEdge(label_t src_label, vid_t src_lid, const Schema& MutablePropertyFragment::schema() const { return schema_; } +Schema& MutablePropertyFragment::mutable_schema() { return schema_; } + void MutablePropertyFragment::Serialize(const std::string& prefix) { std::string data_dir = prefix + "/data"; if (!std::filesystem::exists(data_dir)) { diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.h b/flex/storages/rt_mutable_graph/mutable_property_fragment.h index e530e7078909..c65d97802aea 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.h +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.h @@ -45,6 +45,10 @@ class MutablePropertyFragment { const Schema& schema() const; + Schema& mutable_schema(); + + void Clear(); + void Serialize(const std::string& prefix); void Deserialize(const std::string& prefix); diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index aedba66ada70..3479011c0b68 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -22,6 +22,22 @@ namespace gs { Schema::Schema() = default; Schema::~Schema() = default; +void Schema::Clear() { + vlabel_indexer_.Clear(); + elabel_indexer_.Clear(); + vproperties_.clear(); + vprop_names_.clear(); + v_primary_keys_.clear(); + vprop_storage_.clear(); + eproperties_.clear(); + eprop_names_.clear(); + ie_strategy_.clear(); + oe_strategy_.clear(); + max_vnum_.clear(); + plugin_name_to_path_and_id_.clear(); + plugin_dir_.clear(); +} + void Schema::add_vertex_label( const std::string& label, const std::vector& property_types, const std::vector& property_names, @@ -242,24 +258,25 @@ Schema::get_vertex_primary_key(label_t index) const { return v_primary_keys_.at(index); } +// Note that plugin_dir_ and plugin_name_to_path_and_id_ are not serialized. void Schema::Serialize(std::unique_ptr& writer) { vlabel_indexer_.Serialize(writer); elabel_indexer_.Serialize(writer); grape::InArchive arc; arc << v_primary_keys_ << vproperties_ << vprop_names_ << vprop_storage_ << eproperties_ << eprop_names_ << ie_strategy_ << oe_strategy_ - << max_vnum_ << plugin_dir_ << plugin_list_; + << max_vnum_; CHECK(writer->WriteArchive(arc)); } +// Note that plugin_dir_ and plugin_name_to_path_and_id_ are not deserialized. void Schema::Deserialize(std::unique_ptr& reader) { vlabel_indexer_.Deserialize(reader); elabel_indexer_.Deserialize(reader); grape::OutArchive arc; CHECK(reader->ReadArchive(arc)); arc >> v_primary_keys_ >> vproperties_ >> vprop_names_ >> vprop_storage_ >> - eproperties_ >> eprop_names_ >> ie_strategy_ >> oe_strategy_ >> - max_vnum_ >> plugin_dir_ >> plugin_list_; + eproperties_ >> eprop_names_ >> ie_strategy_ >> oe_strategy_ >> max_vnum_; } label_t Schema::vertex_label_to_index(const std::string& label) { @@ -716,8 +733,9 @@ static bool parse_edges_schema(YAML::Node node, Schema& schema) { return true; } -static bool parse_schema_config_file(const std::string& path, Schema& schema) { - YAML::Node graph_node = YAML::LoadFile(path); +static bool parse_schema_from_yaml_node(const YAML::Node& graph_node, + Schema& schema, + const std::string& parent_dir = "") { if (!graph_node || !graph_node.IsMap()) { LOG(ERROR) << "graph is not set properly"; return false; @@ -742,13 +760,16 @@ static bool parse_schema_config_file(const std::string& path, Schema& schema) { return false; } } - // get the directory of path - auto parent_dir = std::filesystem::path(path).parent_path().string(); + LOG(INFO) << "Parse stored_procedures"; if (graph_node["stored_procedures"]) { auto stored_procedure_node = graph_node["stored_procedures"]; - auto directory = stored_procedure_node["directory"].as(); + std::string directory = "plugins"; // default plugin directory + if (stored_procedure_node["directory"]) { + directory = stored_procedure_node["directory"].as(); + } // check is directory + LOG(INFO) << "Parse directory: " << directory; if (!std::filesystem::exists(directory)) { LOG(ERROR) << "plugin directory - " << directory << " not found, try with parent dir:" << parent_dir; @@ -759,60 +780,112 @@ static bool parse_schema_config_file(const std::string& path, Schema& schema) { } } schema.SetPluginDir(directory); - std::vector files_got; - if (!get_sequence(stored_procedure_node, "enable_lists", files_got)) { + std::vector plugin_name_or_path; + if (!get_sequence(stored_procedure_node, "enable_lists", + plugin_name_or_path)) { LOG(ERROR) << "stored_procedures is not set properly"; return true; } - std::vector all_procedure_yamls = get_yaml_files(directory); - std::vector all_procedure_names; - { - // get all procedure names - for (auto& f : all_procedure_yamls) { - YAML::Node procedure_node = YAML::LoadFile(f); - if (!procedure_node || !procedure_node.IsMap()) { - LOG(ERROR) << "procedure is not set properly"; - return false; - } - std::string procedure_name; - if (!get_scalar(procedure_node, "name", procedure_name)) { - LOG(ERROR) << "name is not set properly for " << f; - return false; - } - all_procedure_names.push_back(procedure_name); - } - } - for (auto& f : files_got) { - auto real_file = directory + "/" + f; - if (!std::filesystem::exists(real_file)) { - LOG(ERROR) << "plugin - " << real_file << " file not found..."; - // it seems that f is not the filename, but the plugin name, try to find - // the plugin in the directory - if (std::find(all_procedure_names.begin(), all_procedure_names.end(), - f) == all_procedure_names.end()) { - LOG(ERROR) << "plugin - " << f << " not found..."; - } else { - VLOG(1) << "plugin - " << f << " found..."; - schema.EmplacePlugin(f); - } - } else { - schema.EmplacePlugin(std::filesystem::canonical(real_file)); - } + // plugin_name_or_path contains the plugin name or path. + // for path, we just use it as the plugin name, and emplace into the map, + // for name, we try to find the plugin in the directory + if (!schema.EmplacePlugins(plugin_name_or_path)) { + LOG(ERROR) << "Fail to emplace all plugins"; } } return true; } +static bool parse_schema_config_file(const std::string& path, Schema& schema) { + YAML::Node graph_node = YAML::LoadFile(path); + // get the directory of path + auto parent_dir = std::filesystem::path(path).parent_path().string(); + + return parse_schema_from_yaml_node(graph_node, schema, parent_dir); +} + } // namespace config_parsing -const std::vector& Schema::GetPluginsList() const { - return plugin_list_; +const std::unordered_map>& +Schema::GetPlugins() const { + return plugin_name_to_path_and_id_; } -void Schema::EmplacePlugin(const std::string& plugin) { - plugin_list_.emplace_back(plugin); +bool Schema::EmplacePlugins( + const std::vector& plugin_paths_or_names) { + std::vector all_procedure_yamls; + if (!plugin_dir_.empty()) { + all_procedure_yamls = get_yaml_files(plugin_dir_); + } + + std::vector all_procedure_names; + + uint8_t cur_plugin_id = RESERVED_PLUGIN_NUM; + std::unordered_set plugin_names; + for (auto& f : plugin_paths_or_names) { + if (std::filesystem::exists(f)) { + plugin_name_to_path_and_id_.emplace(f, + std::make_pair(f, cur_plugin_id++)); + } else { + auto real_file = plugin_dir_ + "/" + f; + if (!std::filesystem::exists(real_file)) { + LOG(ERROR) << "plugin - " << real_file + << " file not found, try to " + "find the plugin in the directory..."; + // it seems that f is not the filename, but the plugin name, try to + // find the plugin in the directory + VLOG(1) << "plugin - " << f << " found..."; + plugin_names.insert(f); + } else { + plugin_name_to_path_and_id_.emplace( + real_file, std::make_pair(real_file, cur_plugin_id++)); + } + } + } + // if there exists any plugins specified by name, add them + // Iterator over the map, and add the plugin path and name to the vector + for (auto cur_yaml : all_procedure_yamls) { + YAML::Node root; + try { + root = YAML::LoadFile(cur_yaml); + } catch (std::exception& e) { + LOG(ERROR) << "Exception when loading from yaml: " << cur_yaml << ":" + << e.what(); + continue; + } + if (root["name"] && root["library"]) { + std::string name = root["name"].as(); + std::string path = root["library"].as(); + if (plugin_names.find(name) != plugin_names.end()) { + if (plugin_name_to_path_and_id_.find(name) != + plugin_name_to_path_and_id_.end()) { + LOG(ERROR) << "Plugin " << name << " already exists, skip"; + } else if (!std::filesystem::exists(path)) { + path = plugin_dir_ + "/" + path; + if (!std::filesystem::exists(path)) { + LOG(ERROR) << "plugin - " << name << "not found from " << path; + } else { + plugin_name_to_path_and_id_.emplace( + name, std::make_pair(path, cur_plugin_id++)); + } + } else { + plugin_name_to_path_and_id_.emplace( + name, std::make_pair(path, cur_plugin_id++)); + } + } else { + VLOG(10) + << "Skip load plugin " << name << ", found in " << cur_yaml + << ", but not specified in the enable_lists in graph schema yaml"; + } + } else { + LOG(ERROR) << "Invalid yaml file: " << cur_yaml + << ", name or library not found."; + } + } + LOG(INFO) << "Load " << plugin_name_to_path_and_id_.size() << " plugins"; + return true; } void Schema::SetPluginDir(const std::string& dir) { plugin_dir_ = dir; } @@ -891,4 +964,13 @@ Schema Schema::LoadFromYaml(const std::string& schema_config) { return schema; } +Result Schema::LoadFromYamlNode(const YAML::Node& schema_yaml_node) { + Schema schema; + if (!config_parsing::parse_schema_from_yaml_node(schema_yaml_node, schema)) { + return Result( + Status(StatusCode::InvalidSchema, "Failed to parse schema"), schema); + } + return schema; +} + } // namespace gs diff --git a/flex/storages/rt_mutable_graph/schema.h b/flex/storages/rt_mutable_graph/schema.h index e592ae77c84d..91f2d0fd3e59 100644 --- a/flex/storages/rt_mutable_graph/schema.h +++ b/flex/storages/rt_mutable_graph/schema.h @@ -20,16 +20,22 @@ #include "flex/storages/rt_mutable_graph/types.h" #include "flex/utils/id_indexer.h" #include "flex/utils/property/table.h" +#include "flex/utils/result.h" #include "flex/utils/yaml_utils.h" namespace gs { class Schema { public: + // How many built-in plugins are there. + // Currently only one builtin plugin, SERVER_APP is supported. + static constexpr uint8_t RESERVED_PLUGIN_NUM = 1; using label_type = label_t; Schema(); ~Schema(); + void Clear(); + void add_vertex_label( const std::string& label, const std::vector& property_types, const std::vector& property_names, @@ -143,11 +149,15 @@ class Schema { static Schema LoadFromYaml(const std::string& schema_config); + static Result LoadFromYamlNode(const YAML::Node& schema_node); + bool Equals(const Schema& other) const; - const std::vector& GetPluginsList() const; + // Return the map from plugin name to plugin id + const std::unordered_map>& + GetPlugins() const; - void EmplacePlugin(const std::string& plugin_name); + bool EmplacePlugins(const std::vector& plugin_paths_or_names); void SetPluginDir(const std::string& plugin_dir); @@ -173,7 +183,9 @@ class Schema { std::map oe_strategy_; std::map ie_strategy_; std::vector max_vnum_; - std::vector plugin_list_; + std::unordered_map> + plugin_name_to_path_and_id_; // key is plugin name, value is plugin path + // and plugin id std::string plugin_dir_; }; diff --git a/flex/tests/hqps/admin_http_test.cc b/flex/tests/hqps/admin_http_test.cc new file mode 100644 index 000000000000..48a7a913bd3a --- /dev/null +++ b/flex/tests/hqps/admin_http_test.cc @@ -0,0 +1,422 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "flex/proto_generated_gie/stored_procedure.pb.h" +#include "flex/third_party/httplib.h" +#include "yaml-cpp/yaml.h" + +#include "glog/logging.h" + +static constexpr const char* CREATE_PROCEDURE_PAYLOAD_TEMPLATE = + "{\"bound_graph\": \"%1%\"," + "\"description\": \"test procedure\"," + "\"enable\": %2%," + "\"name\": \"%3%\"," + "\"query\": \"%4%\"," + "\"type\": \"cypher\"}"; + +std::string generate_start_service_payload(const std::string& graph_name) { + boost::format formater("{\"graph_name\": \"%1%\"}"); + return (formater % graph_name).str(); +} + +std::string get_file_name_from_path(const std::string& file_path) { + auto file_name = file_path.substr(file_path.find_last_of('/') + 1); + // remove extension + file_name = file_name.substr(0, file_name.find_last_of('.')); + return file_name; +} + +std::string generate_call_procedure_payload(const std::string& graph_name, + const std::string& procedure_name) { + query::Query query; + query.mutable_query_name()->set_name(procedure_name); + return query.SerializeAsString(); +} + +std::string generate_update_procedure_payload(const std::string& procedure_name, + bool enabled) { + boost::format formater("{\"enable\": %1%, \"name\": \"%2%\"}"); + return (formater % (enabled ? "true" : "false") % procedure_name).str(); +} + +std::string generate_create_procedure_payload(const std::string& graph_name, + const std::string& procedure_path, + bool enabled) { + boost::format formater(CREATE_PROCEDURE_PAYLOAD_TEMPLATE); + // read from procedure_path and result into a one-line string + std::ifstream ifs(procedure_path); + std::string query((std::istreambuf_iterator(ifs)), + (std::istreambuf_iterator())); + // replace all \n with space + std::replace(query.begin(), query.end(), '\n', ' '); + // get file name + auto file_name = get_file_name_from_path(procedure_path); + return (formater % graph_name % (enabled ? "true" : "false") % file_name % + query) + .str(); +} + +std::string insert_raw_csv_dir(const std::string& raw_csv_dir, + const std::string& import_file_path) { + // Load the file as yaml, and insert + // ["loading_config"]["data_source"]["location"] + YAML::Node node; + try { + node = YAML::LoadFile(import_file_path); + } catch (YAML::BadFile& e) { + LOG(FATAL) << "load import file failed: " << e.what(); + } + node["loading_config"]["data_source"]["location"] = raw_csv_dir; + YAML::Emitter emitter; + emitter << YAML::DoubleQuoted << YAML::Flow << YAML::BeginSeq << node; + std::string json(emitter.c_str() + 1); + return json; +} + +void run_builtin_graph_test( + httplib::Client& admin_client, httplib::Client& query_client, + const std::string& graph_name, + const std::vector>& queries) { + //-------0. get graph schema-------------------------------- + auto res = admin_client.Get("/v1/graph/" + graph_name + "/schema"); + if (res->status != 200) { + LOG(FATAL) << "get graph schema failed for builtin graph" << graph_name + << ": " << res->body; + } + //-------1. create procedures-------------------------------- + { + for (auto& pair : queries) { + auto query_name = pair.first; + auto query_str = pair.second; + boost::format formater(CREATE_PROCEDURE_PAYLOAD_TEMPLATE); + formater % graph_name % "true" % query_name % query_str; + std::string create_proc_payload0 = formater.str(); + auto res = admin_client.Post("/v1/graph/" + graph_name + "/procedure", + create_proc_payload0, "text/plain"); + CHECK(res->status == 200) << "create procedure failed: " << res->body + << ", for query: " << create_proc_payload0; + LOG(INFO) << "Create procedure: " << create_proc_payload0 + << ",response:" << res->body; + } + } + //-------2. now call procedure should fail + { + for (auto& pair : queries) { + auto query_name = pair.first; + auto query_str = pair.second; + query::Query query; + query.mutable_query_name()->set_name(query_name); + auto res = query_client.Post("/v1/query", query.SerializeAsString(), + "text/plain"); + CHECK(res->status != 200) << "call procedure should fail: " << res->body; + } + } + //-------3.restart service + { + // graph_name is not specified, should restart on current graph. + std::string empty_payload; + auto res = + admin_client.Post("/v1/service/restart", empty_payload, "text/plain"); + CHECK(res->status == 200) << "restart service failed: " << res->body; + } + { + //-----3.1 get all procedures. + auto res = admin_client.Get("/v1/graph/" + graph_name + "/procedure"); + CHECK(res->status == 200) << "get all procedures failed: " << res->body; + LOG(INFO) << "get all procedures response: " << res->body; + } + //------4. now do the query + { + for (auto& pair : queries) { + auto query_name = pair.first; + auto query_str = pair.second; + query::Query query; + query.mutable_query_name()->set_name(query_name); + auto res = query_client.Post("/v1/query", query.SerializeAsString(), + "text/plain"); + CHECK(res->status == 200) + << "call procedure should success: " << res->body + << ", for query: " << query.DebugString(); + } + } + LOG(INFO) << "Pass builtin graph test"; +} + +void run_graph_tests(httplib::Client& cli, const std::string& graph_name, + const std::string& schema_path, + const std::string& import_path, + const std::string& raw_data_dir) { + //-------0. create graph-------------------------------- + // load schema_path to yaml and output yaml as json + YAML::Node node; + try { + node = YAML::LoadFile(schema_path); + } catch (YAML::BadFile& e) { + LOG(FATAL) << "load schema file failed: " << e.what(); + } + + YAML::Emitter emitter; + emitter << YAML::DoubleQuoted << YAML::Flow << YAML::BeginSeq << node; + std::string json(emitter.c_str() + 1); + auto res = cli.Post("/v1/graph/", json, "application/json"); + if (res->status != 200) { + LOG(FATAL) << "create graph failed: " << res->body; + } + auto body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "create graph response: " << body; + + ///----1. get graph schema---------------------------- + res = cli.Get("/v1/graph/" + graph_name + "/schema"); + if (res->status != 200) { + LOG(FATAL) << "get graph schema failed: " << res->body; + } + body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "get graph schema response: " << body; + + //----2. list graph----------------------------------- + res = cli.Get("/v1/graph/"); + if (res->status != 200) { + LOG(FATAL) << "list graph failed: " << res->body; + } + body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "list graph response: " << body; + + //----3. load graph----------------------------------- + res = cli.Post("/v1/graph/" + graph_name + "/dataloading", + insert_raw_csv_dir(raw_data_dir, import_path), "text/plain"); + if (res->status != 200) { + LOG(FATAL) << "load graph failed: " << res->body; + } + body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "load graph response: " << body; +} + +// Create the procedure and call the procedure. +void run_procedure_test(httplib::Client& client, httplib::Client& query_client, + const std::string& graph_name, + const std::vector>& + builtin_graph_queries, + const std::vector& procedures) { + // First create the procedure with disabled state, then update with enabled + // state. + //-----0. get all procedures, should be empty---------------------- + auto res = client.Get("/v1/graph/" + graph_name + "/procedure"); + CHECK(res->status == 200) << "get all procedures failed: " << res->body; + + //-----1. create procedures---------------------------------------- + for (auto& procedure : procedures) { + auto create_proc_payload = + generate_create_procedure_payload(graph_name, procedure, false); + LOG(INFO) << "Creating procedure:" << create_proc_payload; + res = client.Post("/v1/graph/" + graph_name + "/procedure", + create_proc_payload, "text/plain"); + CHECK(res->status == 200) << "create procedure failed: " << res->body + << ", for query: " << create_proc_payload; + LOG(INFO) << "response:" << res->body; + } + //-----2. get all procedures-------------------------------------- + res = client.Get("/v1/graph/" + graph_name + "/procedure"); + CHECK(res->status == 200) << "get all procedures failed: " << res->body; + LOG(INFO) << "get all procedures response: " << res->body; + // Step4: update procedures + for (auto& procedure : procedures) { + auto proc_name = get_file_name_from_path(procedure); + auto update_proc_payload = + generate_update_procedure_payload(proc_name, true); + res = client.Put("/v1/graph/" + graph_name + "/procedure/" + proc_name, + update_proc_payload, "text/plain"); + CHECK(res->status == 200) << "update procedure failed: " << res->body + << ", for query: " << update_proc_payload; + } + + //-----3. start service on new graph----------------------------------- + auto start_service_payload = generate_start_service_payload(graph_name); + res = client.Post("/v1/service/start", start_service_payload, "text/plain"); + CHECK(res->status == 200) << "start service failed: " << res->body + << ", for query: " << start_service_payload; + { + //----3.1 call proc on previous procedures on previous graph, should fail. + for (auto& pair : builtin_graph_queries) { + auto query_name = pair.first; + auto query_str = pair.second; + query::Query query; + query.mutable_query_name()->set_name(query_name); + auto res = query_client.Post("/v1/query", query.SerializeAsString(), + "text/plain"); + CHECK(res->status != 200) + << "call previous procedure on current graph should fail: " + << res->body; + } + } + + //----4. call procedures----------------------------------------------- + for (auto& procedure : procedures) { + auto proc_name = get_file_name_from_path(procedure); + auto call_proc_payload = + generate_call_procedure_payload(graph_name, proc_name); + res = query_client.Post("/v1/query", call_proc_payload, "text/plain"); + CHECK(res->status == 200) << "call procedure failed: " << res->body + << ", for query: " << call_proc_payload; + } + //----5. delete procedure by name----------------------------------------- + if (procedures.size() > 0) { + auto proc_name = get_file_name_from_path(procedures[0]); + res = client.Delete("/v1/graph/" + graph_name + "/procedure/" + proc_name); + CHECK(res->status == 200) << "delete procedure failed: " << res->body; + } + //-----6. call procedure on deleted procedure------------------------------ + // Should return success, since the procedure will be deleted when restart + // the service. + if (procedures.size() > 0) { + auto proc_name = get_file_name_from_path(procedures[0]); + auto call_proc_payload = + generate_call_procedure_payload(graph_name, proc_name); + res = query_client.Post("/v1/query", call_proc_payload, "text/plain"); + CHECK(res->status == 200) << "call procedure failed: " << res->body + << ", for query: " << call_proc_payload; + } + //-----7. get procedure by name-------------------------------------------- + // get the second procedure by name + if (procedures.size() > 1) { + auto proc_name = get_file_name_from_path(procedures[1]); + res = client.Get("/v1/graph/" + graph_name + "/procedure/" + proc_name); + CHECK(res->status == 200) << "get procedure failed: " << res->body; + } +} + +void run_get_node_status(httplib::Client& cli) { + auto res = cli.Get("/v1/node/status"); + if (res->status != 200) { + LOG(FATAL) << "get node status failed: " << res->body; + } + auto body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "get node status response: " << body; + // get service status + res = cli.Get("/v1/service/status"); + if (res->status != 200) { + LOG(FATAL) << "get service status failed: " << res->body; + } + body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "get service status response: " << body; +} + +void test_delete_graph(httplib::Client& cli, const std::string& graph_name) { + auto res = cli.Delete("/v1/graph/" + graph_name); + if (res->status != 200) { + LOG(FATAL) << "delete graph failed: " << res->body; + } + auto body = res->body; + if (body.empty()) { + LOG(FATAL) << "Empty response: "; + } + LOG(INFO) << "delete graph response: " << body; +} + +void remove_graph_if_exists(httplib::Client& cli, + const std::string& graph_name) { + auto res = cli.Get("/v1/graph/" + graph_name + "/schema"); + if (res->status == 200) { + LOG(INFO) << "graph " << graph_name << " exists, delete it"; + test_delete_graph(cli, graph_name); + } +} + +int main(int argc, char** argv) { + if (argc < 6) { + std::cerr << "usage: admin_http_test " + " " + " [procedure_path1 " + "procedure_path2]" + << std::endl; + return -1; + } + std::string url; + const char* url_env = std::getenv("GRAPHSCOPE_IP"); + if (url_env == NULL) { + url = "127.0.0.1"; + } else { + url = url_env; + } + int admin_port = atoi(argv[1]); + int query_port = atoi(argv[2]); + auto schema_path = argv[3]; + auto import_path = argv[4]; + auto raw_data_dir = argv[5]; + std::vector procedure_paths; + + std::string builtin_graph_name = "modern_graph"; + + std::string graph_name; + { + // load yaml from schema_path + YAML::Node node; + try { + node = YAML::LoadFile(schema_path); + } catch (YAML::BadFile& e) { + LOG(ERROR) << "load schema file failed: " << e.what(); + return -1; + } + graph_name = node["name"].as(); + } + LOG(INFO) << "graph name: " << graph_name; + + for (auto i = 6; i < argc; ++i) { + procedure_paths.emplace_back(argv[i]); + } + httplib::Client cli(url, admin_port); + cli.set_connection_timeout(0, 300000); + cli.set_read_timeout(60, 0); + cli.set_write_timeout(60, 0); + httplib::Client cli_query(url, query_port); + cli_query.set_connection_timeout(0, 300000); + cli_query.set_read_timeout(60, 0); + cli_query.set_write_timeout(60, 0); + + remove_graph_if_exists(cli, graph_name); + std::vector> builtin_graph_queries = { + {"query0", "MATCH(a) return COUNT(a);"}}; + run_builtin_graph_test(cli, cli_query, builtin_graph_name, + builtin_graph_queries); + run_graph_tests(cli, graph_name, schema_path, import_path, raw_data_dir); + LOG(INFO) << "run graph tests done"; + run_procedure_test(cli, cli_query, graph_name, builtin_graph_queries, + procedure_paths); + LOG(INFO) << "run procedure tests done"; + run_get_node_status(cli); + test_delete_graph(cli, graph_name); + LOG(INFO) << "test delete graph done"; + return 0; +} \ No newline at end of file diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test.yaml new file mode 100644 index 000000000000..d381d5d71675 --- /dev/null +++ b/flex/tests/hqps/engine_config_test.yaml @@ -0,0 +1,34 @@ +directories: + workspace: /tmp/interactive_workspace + subdirs: + data: data + logs: logs + conf: conf +log_level: INFO +default_graph: ldbc +compute_engine: + type: hiactor + workers: + - localhost:10000 + thread_num_per_worker: 1 +compiler: + planner: + is_on: true + opt: RBO + rules: + - FilterIntoJoinRule + - FilterMatchRule + - NotMatchToAntiJoinRule + endpoint: + default_listen_address: localhost + bolt_connector: + disabled: false + port: 7687 + gremlin_connector: + disabled: true + port: 8182 + query_timeout: 30000 +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 10000 diff --git a/flex/tests/hqps/hqps_admin_test.sh b/flex/tests/hqps/hqps_admin_test.sh new file mode 100644 index 000000000000..fef925977719 --- /dev/null +++ b/flex/tests/hqps/hqps_admin_test.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# Copyright 2020 Alibaba Group Holding Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +FLEX_HOME=${SCRIPT_DIR}/../../ +SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server +GIE_HOME=${FLEX_HOME}/../interactive_engine/ +ADMIN_PORT=7777 +QUERY_PORT=10000 + +# +if [ ! $# -eq 3 ]; then + echo "only receives: $# args, need 3" + echo "Usage: $0 " + exit 1 +fi + +INTERACTIVE_WORKSPACE=$1 +ENGINE_CONFIG_PATH=$2 +GS_TEST_DIR=$3 +if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then + echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists" + mkdir -p ${INTERACTIVE_WORKSPACE} +else + echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} exists" + rm -rf ${INTERACTIVE_WORKSPACE} +fi +if [ ! -f ${ENGINE_CONFIG_PATH} ]; then + echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists" + exit 1 +fi +if [ ! -d ${GS_TEST_DIR} ]; then + echo "GS_TEST_DIR: ${GS_TEST_DIR} not exists" + exit 1 +fi + +GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/movies/movies_schema.yaml +GRAPH_BULK_LOAD_YAML=${GS_TEST_DIR}/flex/movies/movies_import.yaml +RAW_CSV_FILES=${FLEX_HOME}/interactive/examples/movies/ +GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/ +TEST_CYPHER_QUERIES="${FLEX_HOME}/interactive/examples/movies/0_get_user.cypher ${FLEX_HOME}/interactive/examples/movies/5_recommend_rule.cypher" +# rm data dir if exists +if [ -d ${GRAPH_CSR_DATA_DIR} ]; then + rm -rf ${GRAPH_CSR_DATA_DIR} +fi + + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color +err() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2 +} + +info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" +} + + +kill_service(){ + info "Kill Service first" + ps -ef | grep "interactive_server" | awk '{print $2}' | xargs kill -9 || true + sleep 3 + # check if service is killed + info "Kill Service success" +} + +# kill service when exit +trap kill_service EXIT + +# start engine service and load ldbc graph +start_engine_service(){ + #check SERVER_BIN exists + if [ ! -f ${SERVER_BIN} ]; then + err "SERVER_BIN not found" + exit 1 + fi + + cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} --enable-admin-service true " + cmd="${cmd} -w ${INTERACTIVE_WORKSPACE} " + + echo "Start engine service with command: ${cmd}" + ${cmd} & + sleep 5 + #check interactive_server is running, if not, exit + ps -ef | grep "interactive_server" | grep -v grep + + info "Start engine service success" +} + +run_movie_test(){ + echo "run movie test" + pushd ${FLEX_HOME}/build/ + cmd="GLOG_v=10 ./tests/hqps/admin_http_test ${ADMIN_PORT} ${QUERY_PORT} ${GRAPH_SCHEMA_YAML} ${GRAPH_BULK_LOAD_YAML} ${RAW_CSV_FILES} ${TEST_CYPHER_QUERIES}" + echo "Start movie test: ${cmd}" + eval ${cmd} || (err "movie test failed" && exit 1) + info "Finish movie test" + popd +} + +kill_service +start_engine_service +run_movie_test +kill_service + + + + diff --git a/flex/tests/hqps/hqps_cypher_test.sh b/flex/tests/hqps/hqps_cypher_test.sh index a6876c862356..bd4a23d98142 100644 --- a/flex/tests/hqps/hqps_cypher_test.sh +++ b/flex/tests/hqps/hqps_cypher_test.sh @@ -15,20 +15,19 @@ set -e SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) FLEX_HOME=${SCRIPT_DIR}/../../ -SERVER_BIN=${FLEX_HOME}/build/bin/sync_server +SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server GIE_HOME=${FLEX_HOME}/../interactive_engine/ # -if [ ! $# -eq 4 ]; then +if [ ! $# -eq 3 ]; then echo "only receives: $# args, need 4" - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi INTERACTIVE_WORKSPACE=$1 GRAPH_NAME=$2 -GRAPH_BULK_LOAD_YAML=$3 -ENGINE_CONFIG_PATH=$4 +ENGINE_CONFIG_PATH=$3 if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists" exit 1 @@ -46,22 +45,13 @@ if [ ! -f ${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME}/graph.yaml ]; then echo "GRAPH_SCHEMA_FILE: ${BULK_LOAD_FILE} not exists" exit 1 fi -if [ ! -f ${GRAPH_BULK_LOAD_YAML} ]; then - echo "GRAPH_BULK_LOAD_YAML: ${GRAPH_BULK_LOAD_YAML} not exists" - exit 1 -fi if [ ! -f ${ENGINE_CONFIG_PATH} ]; then echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists" exit 1 fi GRAPH_SCHEMA_YAML=${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME}/graph.yaml -GRAPH_CSR_DATA_DIR=${HOME}/csr-data-dir/ -# rm data dir if exists -if [ -d ${GRAPH_CSR_DATA_DIR} ]; then - rm -rf ${GRAPH_CSR_DATA_DIR} -fi - +GRAPH_CSR_DATA_DIR=${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME}/indices RED='\033[0;31m' GREEN='\033[0;32m' @@ -78,7 +68,7 @@ info() { kill_service(){ info "Kill Service first" ps -ef | grep "com.alibaba.graphscope.GraphServer" | awk '{print $2}' | xargs kill -9 || true - ps -ef | grep "sync_server" | awk '{print $2}' | xargs kill -9 || true + ps -ef | grep "interactive_server" | awk '{print $2}' | xargs kill -9 || true sleep 3 # check if service is killed info "Kill Service success" @@ -97,13 +87,13 @@ start_engine_service(){ fi cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} " - cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} -l ${GRAPH_BULK_LOAD_YAML} " + cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} " echo "Start engine service with command: ${cmd}" ${cmd} & sleep 5 - #check sync_server is running, if not, exit - ps -ef | grep "sync_server" | grep -v grep + #check interactive_server is running, if not, exit + ps -ef | grep "interactive_server" | grep -v grep info "Start engine service success" } diff --git a/flex/tests/hqps/match_query.h b/flex/tests/hqps/match_query.h index b8ee5edbebbf..1e1282863741 100644 --- a/flex/tests/hqps/match_query.h +++ b/flex/tests/hqps/match_query.h @@ -16,7 +16,6 @@ #ifndef TESTS_HQPS_MATCH_QUERY_H_ #define TESTS_HQPS_MATCH_QUERY_H_ -#include "flex/engines/hqps_db/app/hqps_app_base.h" #include "flex/engines/hqps_db/core/sync_engine.h" #include "flex/utils/app_utils.h" @@ -52,18 +51,25 @@ struct Query5expr1 { private: }; -class MatchQuery : public HqpsAppBase { +class MatchQuery : public AppBase { public: using GRAPH_INTERFACE = gs::MutableCSRInterface; using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t; public: - results::CollectiveResults Query(const GRAPH_INTERFACE& graph, - Decoder& input) const override { - return Query(graph); + MatchQuery(const GraphDBSession& session) : graph(session) {} + bool Query(Decoder& decoder, Encoder& encoder) override { + // decoding params from decoder, and call real query func + + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } - results::CollectiveResults Query(const GRAPH_INTERFACE& graph) const { + results::CollectiveResults Query() const { using label_id_t = typename GRAPH_INTERFACE::label_id_t; using Engine = SyncEngine; @@ -117,15 +123,17 @@ class MatchQuery : public HqpsAppBase { return Engine::Sink(graph, ctx3, std::array{0, 1}); } + gs::MutableCSRInterface graph; }; -class MatchQuery1 : public HqpsAppBase { +class MatchQuery1 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + MatchQuery1(const GraphDBSession& session) : graph(session) {} + results::CollectiveResults Query() const { auto ctx0 = Engine::template ScanVertex( graph, std::array{0, 1, 2, 3, 4, 5, 6, 7}, Filter()); @@ -154,21 +162,28 @@ class MatchQuery1 : public HqpsAppBase { return Engine::Sink(graph, ctx3, std::array{0}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery2 : public HqpsAppBase { +class MatchQuery2 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery2(const GraphDBSession& session) : graph(session) {} + // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query0expr0()); auto ctx0 = Engine::template ScanVertex( graph, std::array{1, 2, 3, 4, 5, 6, 7}, @@ -210,21 +225,27 @@ class MatchQuery2 : public HqpsAppBase { return Engine::Sink(graph, ctx3, std::array{3}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery3 : public HqpsAppBase { +class MatchQuery3 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery3(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto ctx0 = Engine::template ScanVertex( graph, std::array{1, 2, 3, 4, 5, 6, 7}, Filter()); @@ -278,22 +299,28 @@ class MatchQuery3 : public HqpsAppBase { return Engine::Sink(graph, ctx6, std::array{2}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; // Query4 -class MatchQuery4 : public HqpsAppBase { +class MatchQuery4 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery4(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query0expr0()); auto ctx0 = Engine::template ScanVertex( graph, 0, std::move(expr0)); @@ -374,19 +401,26 @@ class MatchQuery4 : public HqpsAppBase { } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery5 : public HqpsAppBase { +class MatchQuery5 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery5(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query5expr0(), gs::PropertySelector("id")); auto ctx0 = Engine::template ScanVertex( @@ -423,21 +457,27 @@ class MatchQuery5 : public HqpsAppBase { return Engine::Sink(graph, ctx3, std::array{2}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery7 : public HqpsAppBase { +class MatchQuery7 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery7(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto ctx0 = Engine::template ScanVertex( graph, 1, Filter()); @@ -467,21 +507,27 @@ class MatchQuery7 : public HqpsAppBase { return Engine::Sink(graph, ctx5, std::array{3}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery9 : public HqpsAppBase { +class MatchQuery9 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery9(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query0expr0()); auto ctx0 = Engine::template ScanVertex( graph, 4, std::move(expr0)); @@ -510,12 +556,17 @@ class MatchQuery9 : public HqpsAppBase { return Engine::Sink(graph, ctx4, std::array{3}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; // test path expand @@ -543,13 +594,14 @@ struct Query10expr1 { private: }; -class MatchQuery10 : public HqpsAppBase { +class MatchQuery10 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery10(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter( Query10expr0(), gs::PropertySelector("firstName"), gs::PropertySelector("id")); @@ -614,12 +666,17 @@ class MatchQuery10 : public HqpsAppBase { return Engine::Sink(graph, ctx7, std::array{4, 5, 6, 7}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; struct MatchQuery11Expr0 { @@ -643,13 +700,14 @@ struct MatchQuery11Expr1 { private: }; -class MatchQuery11 : public HqpsAppBase { +class MatchQuery11 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery11(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(MatchQuery11Expr0(), gs::PropertySelector("id")); auto ctx0 = Engine::template ScanVertex( @@ -734,21 +792,27 @@ class MatchQuery11 : public HqpsAppBase { return Engine::Sink(graph, ctx4, std::array{0, 1}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery12 : public HqpsAppBase { +class MatchQuery12 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery12(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto ctx0 = Engine::template ScanVertex( graph, 1, Filter()); @@ -762,12 +826,17 @@ class MatchQuery12 : public HqpsAppBase { return res; } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; struct MatchQuery13Expr0 { @@ -794,13 +863,14 @@ struct MatchQuery13Expr1 { private: }; -class MatchQuery13 : public HqpsAppBase { +class MatchQuery13 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery13(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto ctx0 = Engine::template ScanVertex( graph, 1, Filter()); @@ -820,22 +890,28 @@ class MatchQuery13 : public HqpsAppBase { return Engine::Sink(graph, ctx3, std::array{2}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; // Auto generated query class definition -class MatchQuery14 : public HqpsAppBase { +class MatchQuery14 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery14(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query0expr0()); auto ctx0 = Engine::template ScanVertex( graph, 2, std::move(expr0)); @@ -861,21 +937,27 @@ class MatchQuery14 : public HqpsAppBase { return Engine::Sink(graph, ctx2, std::array{2, 3}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; -class MatchQuery15 : public HqpsAppBase { +class MatchQuery15 : public AppBase { public: using Engine = SyncEngine; using label_id_t = typename gs::MutableCSRInterface::label_id_t; using vertex_id_t = typename gs::MutableCSRInterface::vertex_id_t; + MatchQuery15(const GraphDBSession& session) : graph(session) {} // Query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph) const { + results::CollectiveResults Query() const { auto expr0 = gs::make_filter(Query0expr0()); auto ctx0 = Engine::template ScanVertex( graph, std::array{0, 1, 2, 3, 4, 5, 6, 7}, @@ -938,12 +1020,17 @@ class MatchQuery15 : public HqpsAppBase { return Engine::Sink(graph, ctx4, std::array{2, 3}); } // Wrapper query function for query class - results::CollectiveResults Query(const gs::MutableCSRInterface& graph, - Decoder& decoder) const override { + bool Query(Decoder& decoder, Encoder& encoder) override { // decoding params from decoder, and call real query func - return Query(graph); + auto res = Query(); + // dump results to string + std::string res_str = res.SerializeAsString(); + // encode results to encoder + encoder.put_string(res_str); + return true; } + gs::MutableCSRInterface graph; }; } // namespace gs diff --git a/flex/tests/hqps/query_test.cc b/flex/tests/hqps/query_test.cc index 04bfabf5643e..37b2bac6775f 100644 --- a/flex/tests/hqps/query_test.cc +++ b/flex/tests/hqps/query_test.cc @@ -34,7 +34,7 @@ int main(int argc, char** argv) { auto& db = gs::GraphDB::get(); auto schema = gs::Schema::LoadFromYaml(graph_schema); auto loading_config = - gs::LoadingConfig::ParseFromYaml(schema, bulk_load_yaml); + gs::LoadingConfig::ParseFromYamlFile(schema, bulk_load_yaml); db.Init(schema, loading_config, data_dir, 1); auto& sess = gs::GraphDB::get().GetSession(0); @@ -71,7 +71,7 @@ int main(int argc, char** argv) { } { - gs::SampleQuery query; + gs::SampleQuery query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); input_encoder.put_long(19791209300143); @@ -80,12 +80,11 @@ int main(int argc, char** argv) { gs::Encoder output(output_array); gs::Decoder input(encoder_array.data(), encoder_array.size()); - gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish Sample query"; } { - gs::MatchQuery query; + gs::MatchQuery query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -93,12 +92,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery test"; } { - gs::MatchQuery1 query; + gs::MatchQuery1 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -106,12 +105,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - auto res = query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery1 test"; } { - gs::MatchQuery2 query; + gs::MatchQuery2 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -119,12 +118,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery2 test"; } { - gs::MatchQuery3 query; + gs::MatchQuery3 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -132,12 +131,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery3 test"; } { - gs::MatchQuery4 query; + gs::MatchQuery4 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -145,12 +144,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery4 test"; } { - gs::MatchQuery5 query; + gs::MatchQuery5 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -158,12 +157,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery5 test"; } { - gs::MatchQuery7 query; + gs::MatchQuery7 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -171,12 +170,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery7 test"; } { - gs::MatchQuery9 query; + gs::MatchQuery9 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -184,12 +183,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery9 test"; } { - gs::MatchQuery10 query; + gs::MatchQuery10 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -197,12 +196,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery10 test"; } { - gs::MatchQuery11 query; + gs::MatchQuery11 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -210,12 +209,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery11 test"; } { - gs::MatchQuery12 query; + gs::MatchQuery12 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -223,12 +222,12 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery12 test"; } { - gs::MatchQuery14 query; + gs::MatchQuery14 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; @@ -236,22 +235,21 @@ int main(int argc, char** argv) { gs::Decoder input(encoder_array.data(), encoder_array.size()); gs::MutableCSRInterface graph(sess); - query.Query(graph, input); + query.Query(input, output); LOG(INFO) << "Finish MatchQuery14 test"; } { // test PathExpand with multiple edge triplets. - gs::MatchQuery15 query; + gs::MatchQuery15 query(sess); std::vector encoder_array; gs::Encoder input_encoder(encoder_array); std::vector output_array; gs::Encoder output(output_array); gs::Decoder input(encoder_array.data(), encoder_array.size()); - gs::MutableCSRInterface graph(sess); - query.Query(graph, input); - LOG(INFO) << "Finish MatchQuery14 test"; + query.Query(input, output); + LOG(INFO) << "Finish MatchQuery15 test"; } LOG(INFO) << "Finish context test."; diff --git a/flex/tests/hqps/sample_query.h b/flex/tests/hqps/sample_query.h index 109ed7e47222..a51fa9443109 100644 --- a/flex/tests/hqps/sample_query.h +++ b/flex/tests/hqps/sample_query.h @@ -16,7 +16,6 @@ #ifndef TESTS_HQPS_SAMPLE_QUERY_H_ #define TESTS_HQPS_SAMPLE_QUERY_H_ -#include "flex/engines/hqps_db/app/hqps_app_base.h" #include "flex/engines/hqps_db/core/sync_engine.h" #include "flex/utils/app_utils.h" @@ -33,17 +32,21 @@ struct Expression1 { int64_t oid_; }; -class SampleQuery : public HqpsAppBase { +class SampleQuery : public AppBase { public: using GRAPH_INTERFACE = gs::MutableCSRInterface; using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t; public: - results::CollectiveResults Query(const GRAPH_INTERFACE& graph, - Decoder& input) const override { + SampleQuery(const GraphDBSession& session) : graph(session) {} + bool Query(Decoder& input, Encoder& output) override { int64_t id = input.get_long(); int64_t maxDate = input.get_long(); - return Query(graph, id, maxDate); + auto res = Query(graph, id, maxDate); + std::string res_str = res.SerializeAsString(); + // encode results to encoder + output.put_string(res_str); + return true; } results::CollectiveResults Query(const GRAPH_INTERFACE& graph, int64_t id, @@ -103,6 +106,7 @@ class SampleQuery : public HqpsAppBase { std::move(mapper7)}); return Engine::Sink(graph, ctx5); } + gs::MutableCSRInterface graph; }; } // namespace gs #endif // TESTS_HQPS_SAMPLE_QUERY_H_ diff --git a/flex/third_party/nlohmann-json b/flex/third_party/nlohmann-json new file mode 160000 index 000000000000..6eab7a2b187b --- /dev/null +++ b/flex/third_party/nlohmann-json @@ -0,0 +1 @@ +Subproject commit 6eab7a2b187b10b2494e39c1961750bfd1bda500 diff --git a/flex/utils/id_indexer.h b/flex/utils/id_indexer.h index a87d9b19102c..cb03846db923 100644 --- a/flex/utils/id_indexer.h +++ b/flex/utils/id_indexer.h @@ -443,6 +443,14 @@ class IdIndexer : public IdIndexerBase { ConvertAny::to(oid, oid_); return get_index(oid_, lid); } + void Clear() { + keys_.clear(); + indices_.clear(); + distances_.clear(); + num_elements_ = 0; + num_slots_minus_one_ = 0; + hash_policy_.reset(); + } size_t entry_num() const { return distances_.size(); } diff --git a/flex/utils/result.cc b/flex/utils/result.cc new file mode 100644 index 000000000000..f2254ef8b36b --- /dev/null +++ b/flex/utils/result.cc @@ -0,0 +1,32 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/utils/result.h" + +namespace gs { +Status::Status() noexcept : error_code_(StatusCode::UninitializedStatus) {} + +Status::Status(StatusCode error_code) noexcept : error_code_(error_code) {} + +Status::Status(StatusCode error_code, std::string&& error_msg) noexcept + : error_code_(error_code), error_msg_(std::move(error_msg)) {} + +std::string Status::error_message() const { return error_msg_; } + +bool Status::ok() const { return error_code_ == StatusCode::OK; } + +Status Status::OK() { return Status(StatusCode::OK); } + +} // namespace gs diff --git a/flex/utils/result.h b/flex/utils/result.h new file mode 100644 index 000000000000..0cc7d274d60a --- /dev/null +++ b/flex/utils/result.h @@ -0,0 +1,105 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UTILS_RESULT_H_ +#define UTILS_RESULT_H_ + +#include +#include +#include +#include + +#include "glog/logging.h" + +namespace gs { +enum class StatusCode { + OK = 0, + InValidArgument = 1, + UnsupportedOperator = 2, + AlreadyExists = 3, + NotExists = 4, + CodegenError = 5, + UninitializedStatus = 6, + InvalidSchema = 7, + PermissionError = 8, + IllegalOperation = 9, + InternalError = 10, + InvalidImportFile = 11, + IOError = 12, + NotFound = 13, + QueryFailed = 14, +}; + +class Status { + public: + Status() noexcept; + Status(StatusCode error_code) noexcept; + Status(StatusCode error_code, std::string&& error_msg) noexcept; + + bool ok() const; + std::string error_message() const; + + static Status OK(); + + private: + StatusCode error_code_; + std::string error_msg_; +}; + +// Define a class with name Result, which is a template class +// Stores the result of a function that may fail. +// If the function succeeds, the result contains the value returned by the +// function. If the function fails, the result contains the error message. The +// result is always valid and can be queried for success or failure. If the +// result is successful, the value can be obtained by calling the value() +// method. If the result fails, the error message can be obtained by calling the +// error() method. The result can be converted to bool, and the result is true +// if the result is successful. The result can be converted to bool, and the +// result is false if the result fails. +template +class Result { + public: + using ValueType = T; + Result() : status_(StatusCode::UninitializedStatus) {} + Result(const ValueType& value) : status_(StatusCode::OK), value_(value) {} + Result(ValueType&& value) + : status_(StatusCode::OK), value_(std::move(value)) {} + Result(const Status& status, ValueType&& value) + : status_(status), value_(std::move(value)) {} + + Result(const Status& status) : status_(status) {} + + Result(const Status& status, const ValueType& value) + : status_(status), value_(value) {} + + Result(StatusCode code, const std::string& error_msg, const ValueType& value) + : status_(code, error_msg), value_(value) {} + + Result(StatusCode code, std::string&& error_msg, const ValueType& value) + : status_(code, std::move(error_msg)), value_(value) {} + + bool ok() const noexcept { return status_.ok(); } + + const Status& status() const noexcept { return status_; } + ValueType& value() noexcept { return value_; } + + private: + Status status_; + ValueType value_; +}; + +} // namespace gs + +#endif // UTILS_RESULT_H_ \ No newline at end of file diff --git a/flex/utils/service_utils.cc b/flex/utils/service_utils.cc new file mode 100644 index 000000000000..9fc926808e79 --- /dev/null +++ b/flex/utils/service_utils.cc @@ -0,0 +1,147 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/utils/service_utils.h" + +namespace gs { + +static unsigned long long lastTotalUser, lastTotalUserLow, lastTotalSys, + lastTotalIdle; + +FlexException::FlexException(std::string&& error_msg) + : std::exception(), _err_msg(error_msg) {} + +FlexException::~FlexException() {} + +const char* FlexException::what() const noexcept { return _err_msg.c_str(); } + +// get current executable's directory +std::string get_current_dir() { + char buf[1024]; + int dirfd = open("/proc/self/", O_RDONLY | O_DIRECTORY); + if (dirfd == -1) { + // Handle error + } + + ssize_t len = readlinkat(dirfd, "exe", buf, sizeof(buf) - 1); + if (len == -1) { + // Handle error + } + buf[len] = '\0'; + close(dirfd); + std::string exe_path(buf); + return exe_path.substr(0, exe_path.rfind('/')); +} + +std::string find_codegen_bin() { + // first check whether flex_home env exists + std::string flex_home; + std::string codegen_bin; + char* flex_home_char = getenv("FLEX_HOME"); + if (flex_home_char == nullptr) { + // infer flex_home from current binary' directory + // get the path of current binary + std::string flex_home_str = get_current_dir(); + // usr/loca/bin/ + flex_home_str = flex_home_str.substr(0, flex_home_str.find_last_of("/")); + // usr/local/ + + LOG(INFO) << "infer flex_home as installed, flex_home: " << flex_home_str; + // check codege_bin_path exists + codegen_bin = flex_home_str + "/bin/" + CODEGEN_BIN; + // if flex_home exists, return flex_home + if (std::filesystem::exists(codegen_bin)) { + return codegen_bin; + } else { + // if not found, try as if it is in build directory + // flex/build/ + flex_home_str = flex_home_str.substr(0, flex_home_str.find_last_of("/")); + // flex/ + LOG(INFO) << "infer flex_home as build, flex_home: " << flex_home_str; + codegen_bin = flex_home_str + "/bin/" + CODEGEN_BIN; + if (std::filesystem::exists(codegen_bin)) { + return codegen_bin; + } else { + LOG(FATAL) << "codegen bin not exists: "; + return ""; + } + } + } else { + flex_home = std::string(flex_home_char); + LOG(INFO) << "flex_home env exists, flex_home: " << flex_home; + codegen_bin = flex_home + "/bin/" + CODEGEN_BIN; + if (std::filesystem::exists(codegen_bin)) { + return codegen_bin; + } else { + LOG(FATAL) << "codegen bin not exists: "; + return ""; + } + } +} + +std::pair get_total_physical_memory_usage() { + struct sysinfo memInfo; + + sysinfo(&memInfo); + uint64_t total_mem = memInfo.totalram; + total_mem *= memInfo.mem_unit; + + uint64_t phy_mem_used = memInfo.totalram - memInfo.freeram; + phy_mem_used *= memInfo.mem_unit; + return std::make_pair(phy_mem_used, total_mem); +} + +void init_cpu_usage_watch() { + FILE* file = fopen("/proc/stat", "r"); + fscanf(file, "cpu %llu %llu %llu %llu", &lastTotalUser, &lastTotalUserLow, + &lastTotalSys, &lastTotalIdle); + fclose(file); +} + +std::pair get_current_cpu_usage() { + double used; + FILE* file; + unsigned long long totalUser, totalUserLow, totalSys, totalIdle, total; + + file = fopen("/proc/stat", "r"); + fscanf(file, "cpu %llu %llu %llu %llu", &totalUser, &totalUserLow, &totalSys, + &totalIdle); + fclose(file); + + if (totalUser < lastTotalUser || totalUserLow < lastTotalUserLow || + totalSys < lastTotalSys || totalIdle < lastTotalIdle) { + // Overflow detection. Just skip this value. + used = total = 0.0; + } else { + total = (totalUser - lastTotalUser) + (totalUserLow - lastTotalUserLow) + + (totalSys - lastTotalSys); + used = total; + total += (totalIdle - lastTotalIdle); + } + + lastTotalUser = totalUser; + lastTotalUserLow = totalUserLow; + lastTotalSys = totalSys; + lastTotalIdle = totalIdle; + + return std::make_pair(used, total); +} + +std::string memory_to_mb_str(uint64_t mem_bytes) { + double mem_mb = mem_bytes / 1024.0 / 1024.0; + return std::to_string(mem_mb) + "MB"; +} + +} // namespace gs diff --git a/flex/utils/service_utils.h b/flex/utils/service_utils.h new file mode 100644 index 000000000000..bd698b054ed0 --- /dev/null +++ b/flex/utils/service_utils.h @@ -0,0 +1,68 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SERVICE_UTILS_H +#define SERVICE_UTILS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flex/utils/result.h" +#include "flex/utils/yaml_utils.h" +#include "nlohmann/json.hpp" + +#include + +namespace gs { + +static constexpr const char* CODEGEN_BIN = "load_plan_and_gen.sh"; +static constexpr const char* GRAPH_LOADER_BIN = "graph_loader"; + +class FlexException : public std::exception { + public: + explicit FlexException(std::string&& error_msg); + ~FlexException() override; + + const char* what() const noexcept override; + + private: + std::string _err_msg; +}; + +// Get the directory of the current executable +std::string get_current_dir(); + +std::string find_codegen_bin(); + +std::pair get_total_physical_memory_usage(); + +void init_cpu_usage_watch(); + +std::pair get_current_cpu_usage(); + +std::string memory_to_mb_str(uint64_t mem_bytes); + +} // namespace gs + +#endif // SERVICE_UTILS_H \ No newline at end of file diff --git a/flex/utils/yaml_utils.cc b/flex/utils/yaml_utils.cc index 2413e3b07fab..5b27f0e600fc 100644 --- a/flex/utils/yaml_utils.cc +++ b/flex/utils/yaml_utils.cc @@ -29,4 +29,30 @@ std::vector get_yaml_files(const std::string& plugin_dir) { return res_yaml_files; } +Result get_string_from_yaml(const std::string& file_path) { + try { + YAML::Node config = YAML::LoadFile(file_path); + // output config to string + return get_string_from_yaml(config); + } catch (const YAML::BadFile& e) { + return Result(Status{StatusCode::IOError, e.what()}); + } catch (const YAML::ParserException& e) { + return Result(Status{StatusCode::IOError, e.what()}); + } catch (const YAML::BadConversion& e) { + return Result(Status{StatusCode::IOError, e.what()}); + } +} + +Result get_string_from_yaml(const YAML::Node& node) { + try { + // output config to string + YAML::Emitter emitter; + emitter << YAML::DoubleQuoted << YAML::Flow << YAML::BeginSeq << node; + std::string json(emitter.c_str() + 1); + return Result(Status{StatusCode::OK, "Success"}, json); + } catch (...) { + return Result( + Status{StatusCode::IOError, "Failed to convert yaml to json"}); + } +} } // namespace gs diff --git a/flex/utils/yaml_utils.h b/flex/utils/yaml_utils.h index fc4572fa1c40..56d5d9282799 100644 --- a/flex/utils/yaml_utils.h +++ b/flex/utils/yaml_utils.h @@ -20,6 +20,7 @@ #include #include #include +#include "flex/utils/result.h" #include "glog/logging.h" @@ -27,6 +28,10 @@ namespace gs { std::vector get_yaml_files(const std::string& plugin_dir); +Result get_string_from_yaml(const std::string& file_path); + +Result get_string_from_yaml(const YAML::Node& yaml_node); + namespace config_parsing { template bool get_scalar(YAML::Node node, const std::string& key, T& value) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 460e0ad2fc0b..b58f7aad881c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -51,6 +51,11 @@ public class YamlConfigs extends Configs { .put( "graph.schema", (Configs configs) -> { + // if System.properties contains graph.schema, use it + String schema = System.getProperty("graph.schema"); + if (schema != null) { + return schema; + } String workspace = configs.get("directories.workspace"); String subdir = configs.get("directories.subdirs.data"); String graphName = configs.get("default_graph"); @@ -121,7 +126,7 @@ public class YamlConfigs extends Configs { if (type == null || !type.equals("pegasus")) { return null; } else { - return configs.get("compute_engine.worker_num"); + return configs.get("compute_engine.thread_num_per_worker"); } }) .put( @@ -151,7 +156,7 @@ public class YamlConfigs extends Configs { if (type == null || !type.equals("pegasus")) { return null; } else { - String hosts = configs.get("compute_engine.hosts"); + String hosts = configs.get("compute_engine.workers"); if (hosts != null) { return hosts.replace("[", "").replace("]", ""); } @@ -165,11 +170,13 @@ public class YamlConfigs extends Configs { if (type == null || !type.equals("hiactor")) { return null; } else { - String hosts = configs.get("compute_engine.hosts"); - if (hosts != null) { + String port = configs.get("http_service.query_port"); + String address = configs.get("http_service.default_listen_address"); + if (port != null && address != null) { + String hosts = address + ":" + port; return hosts.replace("[", "").replace("]", ""); } - return hosts; + return null; } }) .put( diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml index 44ddcf39d4db..d6c6d0398bff 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml @@ -11,8 +11,9 @@ default_graph: modern # configure the graph to be loaded while starting the ser # may include other configuration items of other engines compute_engine: type: hiactor # [hiactor|pegasus] hiactor is for high-qps scenario, pegasus is for bi scenario - hosts: - - localhost:8001 + workers: + - localhost:10000 + thread_num_per_worker: 1 compiler: planner: is_on: true @@ -29,3 +30,7 @@ compiler: port: 8003 # default 7687 query_timeout: 200 # query timeout in milliseconds, default 2000 calcite_default_charset: UTF-8 +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 8001 \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml index c8f00f4380f3..6a18c3b86e3c 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml @@ -11,10 +11,10 @@ default_graph: modern # configure the graph to be loaded while starting the ser # may include other configuration items of other engines compute_engine: type: pegasus # [hiactor|pegasus] hiactor is for high-qps scenario, pegasus is for bi scenario - hosts: + workers: - localhost:8001 - localhost:8005 - worker_num: 3 + thread_num_per_worker: 3 batch_size: 2048 output_capacity: 18