fix(interactive): Fix building flex with -DBUILD_HQPS=OFF

.github/workflows/flex.yml

@@ -50,6 +50,28 @@ jobs:
         mkdir build && cd build # only test default build
         cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_DOC=OFF && sudo make -j 4
 
+  # test the different combination of cmake options: -DBUILD_HQPS=ON/OFF -DBUILD_TEST=ON/OFF, -DBUILD_ODPS_FRAGMENT_LOADER=ON/OFF
+  test-cmake-options:
+    runs-on: ubuntu-20.04
+    container:
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
+    strategy:
+      matrix:
+        BUILD_HQPS: [ON, OFF]
+        BUILD_TEST: [ON, OFF]
+        BUILD_ODPS_FRAGMENT_LOADER: [ON, OFF]
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex
+        git submodule update --init
+        mkdir build && cd build
+        cmake .. -DBUILD_HQPS=${{ matrix.BUILD_HQPS }} -DBUILD_TEST=${{ matrix.BUILD_TEST }} \
+                 -DBUILD_ODPS_FRAGMENT_LOADER=${{ matrix.BUILD_ODPS_FRAGMENT_LOADER }}
+        sudo make -j4
+
   test-flex:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
@@ -72,9 +94,8 @@ jobs:
       env:
         HOME: /home/graphscope/
       run: |
-        cd ${GITHUB_WORKSPACE}/
-        git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
+        git submodule update --init
         mkdir build && cd build
         cmake .. && sudo make -j$(nproc)
 

.github/workflows/hqps-db-ci.yml

@@ -170,6 +170,17 @@ jobs:
           eval ${cmd}
         done
 
+    -name: Test cypher procedure generation
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/bin
+        ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \
+          --ir_conf=/workspaces/GraphScope/flex/tests/hqps/engine_config_test.yaml \
+          --graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
+
+        ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
+          --ir_conf=/workspaces/GraphScope/flex/tests/hqps/engine_config_test.yaml \
+          --graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
+
     - name: Run End-to-End cypher adhoc ldbc query test
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest

flex/CMakeLists.txt

@@ -17,6 +17,14 @@ option(BUILD_ODPS_FRAGMENT_LOADER "Whether to build odps fragment loader" ON)
 option(MONITOR_SESSIONS "Whether monitor sessions" OFF)
 option(ENABLE_HUGEPAGE "Whether to use hugepages when open mmap array in memory" OFF)
 
+#print options
+message(STATUS "Build HighQPS Engine: ${BUILD_HQPS}")
+message(STATUS "Build test: ${BUILD_TEST}")
+message(STATUS "Build doc: ${BUILD_DOC}")
+message(STATUS "Build odps fragment loader: ${BUILD_ODPS_FRAGMENT_LOADER}")
+message(STATUS "Monitor sessions: ${MONITOR_SESSIONS}")
+message(STATUS "Enable hugepage: ${ENABLE_HUGEPAGE}")
+
 # ------------------------------------------------------------------------------
 # cmake configs
 # ------------------------------------------------------------------------------
@@ -31,6 +39,11 @@ set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
+if (BUILD_HQPS)
+    message("Build HighQPS Engine")
+    add_definitions(-DBUILD_HQPS)
+endif ()
+
 if (MONITOR_SESSIONS)
     message("Monitor sessions is enabled")
     add_definitions(-DMONITOR_SESSIONS)

flex/bin/load_plan_and_gen.sh

@@ -276,7 +276,7 @@ compile_hqps_so() {
   if [ ! -z ${CMAKE_C_COMPILER} ]; then
     cmd="${cmd} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
   fi
-  info "Cmake command = ${cmd}"
+  info "CMake command = ${cmd}"
   info "---------------------------"
   eval ${cmd}
 

flex/codegen/CMakeLists.txt

@@ -3,6 +3,8 @@ find_package(Boost REQUIRED COMPONENTS system filesystem
     context program_options regex thread)
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 
-add_executable(gen_code_from_plan gen_code_from_plan.cc)
-target_link_libraries(gen_code_from_plan hqps_plan_proto ${GLOG_LIBRARIES}  ${Boost_LIBRARIES})
-install_flex_target(gen_code_from_plan)
+if (BUILD_HQPS)
+    add_executable(gen_code_from_plan gen_code_from_plan.cc)
+    target_link_libraries(gen_code_from_plan hqps_plan_proto ${GLOG_LIBRARIES}  ${Boost_LIBRARIES})
+    install_flex_target(gen_code_from_plan)
+endif()

flex/codegen/src/hqps/hqps_scan_builder.h

@@ -199,6 +199,9 @@ class ScanOpBuilder {
       VLOG(10) << "receive param const in index predicate: "
                << dyn_param_pb.DebugString();
       ctx_.AddParameterVar(param_const);
+      // set to oid_ and oid_type_name_
+      oid_ = param_const.var_name;
+      oid_type_name_ = data_type_2_string(param_const.type);
     }
 
     return *this;

flex/codegen/src/hqps_generator.h

@@ -71,7 +71,9 @@ static constexpr const char* QUERY_TEMPLATE_STR =
     "    // dump results to string\n"
     "    std::string res_str = res.SerializeAsString();\n"
     "    // encode results to encoder\n"
-    "    encoder.put_string(res_str);\n"
+    "    if (!res_str.empty()){\n"
+    "      encoder.put_string_view(res_str);\n"
+    "    }\n"
     "    return true;\n"
     "  }\n"
     "  //private members\n"
@@ -227,12 +229,16 @@ class QueryGenerator {
           CHECK(param_vars[i] == param_vars[i - 1]);
           continue;
         } else {
-          ss << ", " << data_type_2_string(param_vars[i].type) << " "
-             << param_vars[i].var_name;
+          ss << data_type_2_string(param_vars[i].type) << " "
+             << param_vars[i].var_name << ",";
         }
       }
     }
-    return ss.str();
+    auto str = ss.str();
+    if (str.size() > 0) {
+      str.pop_back();  // remove the last comma
+    }
+    return str;
   }
 
   // implement the function that overrides the base class.
@@ -265,9 +271,6 @@ class QueryGenerator {
     std::string param_vars_decoding, param_vars_concat_str;
     {
       std::stringstream ss;
-      if (param_names.size() > 0) {
-        ss << ",";
-      }
       for (size_t i = 0; i < param_names.size(); ++i) {
         ss << param_names[i];
         if (i != param_names.size() - 1) {

flex/engines/graph_db/CMakeLists.txt

@@ -1,10 +1,16 @@
 file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc"
   	                             "${CMAKE_CURRENT_SOURCE_DIR}/database/*.cc")
+if (NOT BUILD_HQPS)
+        list(FILTER GRAPH_DB_SRC_FILES EXCLUDE REGEX ".*hqps_app*.")
+endif()
 
 add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES})
 
 target_include_directories(flex_graph_db PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
-target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils hqps_plan_proto ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+if (BUILD_HQPS)
+        target_link_libraries(flex_graph_db hqps_plan_proto)
+endif()
 install_flex_target(flex_graph_db)
 
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h
@@ -22,4 +28,8 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h
 
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/app/app_base.h
         DESTINATION include/flex/engines/graph_db/app)
+if (BUILD_HQPS)
+        install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/app/hqps_app.h
+                DESTINATION include/flex/engines/graph_db/app)
+endif()
 

flex/engines/graph_db/app/hqps_app.cc

@@ -0,0 +1,127 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flex/engines/graph_db/app/hqps_app.h"
+#include "flex/proto_generated_gie/stored_procedure.pb.h"
+
+namespace gs {
+
+void put_argment(gs::Encoder& encoder, const query::Argument& argment) {
+  auto& value = argment.value();
+  auto item_case = value.item_case();
+  switch (item_case) {
+  case common::Value::kI32:
+    encoder.put_int(value.i32());
+    break;
+  case common::Value::kI64:
+    encoder.put_long(value.i64());
+    break;
+  case common::Value::kF64:
+    encoder.put_double(value.f64());
+    break;
+  case common::Value::kStr:
+    encoder.put_string(value.str());
+    break;
+  default:
+    LOG(ERROR) << "Not recognizable param type" << static_cast<int>(item_case);
+  }
+}
+
+HQPSAdhocApp::HQPSAdhocApp(GraphDBSession& graph) : graph_(graph) {}
+
+bool HQPSAdhocApp::Query(Decoder& input, Encoder& output) {
+  if (input.size() <= 4) {
+    LOG(ERROR) << "Invalid input for HQPSAdhocApp, input size: "
+               << input.size();
+    return false;
+  }
+  std::string input_lib_path = std::string(input.get_string());
+  auto app_factory = std::make_shared<SharedLibraryAppFactory>(input_lib_path);
+  AppWrapper app_wrapper;  // wrapper should be destroyed before the factory
+
+  if (app_factory) {
+    app_wrapper = app_factory->CreateApp(graph_);
+    if (app_wrapper.app() == NULL) {
+      LOG(ERROR) << "Fail to create app for adhoc query: " << input_lib_path;
+      return false;
+    }
+  } else {
+    LOG(ERROR) << "Fail to evaluate adhoc query: " << input_lib_path;
+    return false;
+  }
+
+  return app_wrapper.app()->Query(input, output);
+}
+
+HQPSProcedureApp::HQPSProcedureApp(GraphDBSession& graph) : graph_(graph) {}
+
+bool HQPSProcedureApp::Query(Decoder& input, Encoder& output) {
+  if (input.size() <= 0) {
+    LOG(ERROR) << "Invalid input for HQPSProcedureApp, input size: "
+               << input.size();
+    return false;
+  }
+  query::Query cur_query;
+  if (!cur_query.ParseFromArray(input.data(), input.size())) {
+    LOG(ERROR) << "Fail to parse query from input content";
+    return false;
+  }
+  auto query_name = cur_query.query_name().name();
+
+  std::vector<char> input_buffer;
+  gs::Encoder input_encoder(input_buffer);
+  auto& args = cur_query.arguments();
+  for (int32_t i = 0; i < args.size(); ++i) {
+    put_argment(input_encoder, args[i]);
+  }
+  VLOG(10) << "Query name: " << query_name << ", args: " << input_buffer.size()
+           << " bytes";
+  gs::Decoder input_decoder(input_buffer.data(), input_buffer.size());
+
+  if (query_name.empty()) {
+    LOG(ERROR) << "Query name is empty";
+    return false;
+  }
+  auto& app_name_to_path_index = graph_.schema().GetPlugins();
+  // get procedure id from name.
+  if (app_name_to_path_index.count(query_name) <= 0) {
+    LOG(ERROR) << "Query name is not registered: " << query_name;
+    return false;
+  }
+
+  // get app
+  auto type = app_name_to_path_index.at(query_name).second;
+  auto app = graph_.GetApp(type);
+  if (!app) {
+    LOG(ERROR) << "Fail to get app for query: " << query_name
+               << ", type: " << type;
+    return false;
+  }
+  return app->Query(input_decoder, output);
+}
+
+HQPSAdhocAppFactory::HQPSAdhocAppFactory() {}
+HQPSAdhocAppFactory::~HQPSAdhocAppFactory() {}
+AppWrapper HQPSAdhocAppFactory::CreateApp(GraphDBSession& graph) {
+  return AppWrapper(new HQPSAdhocApp(graph), NULL);
+}
+
+HQPSProcedureAppFactory::HQPSProcedureAppFactory() {}
+HQPSProcedureAppFactory::~HQPSProcedureAppFactory() {}
+AppWrapper HQPSProcedureAppFactory::CreateApp(GraphDBSession& graph) {
+  return AppWrapper(new HQPSProcedureApp(graph), NULL);
+}
+
+}  // namespace gs

flex/engines/graph_db/app/hqps_app.h

@@ -0,0 +1,70 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ENGINES_GRAPH_DB_APP_HQPS_APP_H_
+#define ENGINES_GRAPH_DB_APP_HQPS_APP_H_
+
+#include "flex/engines/graph_db/app/app_base.h"
+#include "flex/engines/graph_db/database/graph_db_session.h"
+
+namespace gs {
+
+/**
+ * @brief HQPSAdhocApp is a builtin, proxy app used to evaluate adhoc query.
+ */
+class HQPSAdhocApp : public AppBase {
+ public:
+  HQPSAdhocApp(GraphDBSession& graph);
+
+  bool Query(Decoder& input, Encoder& output) override;
+
+ private:
+  GraphDBSession& graph_;
+};
+
+/**
+ * @brief HQPSProcedureApp is a builtin, proxy app used to evaluate procedure
+ * query.
+ */
+class HQPSProcedureApp : public AppBase {
+ public:
+  HQPSProcedureApp(GraphDBSession& graph);
+
+  bool Query(Decoder& input, Encoder& output) override;
+
+ private:
+  GraphDBSession& graph_;
+};
+
+// Factory
+class HQPSAdhocAppFactory : public AppFactoryBase {
+ public:
+  HQPSAdhocAppFactory();
+  ~HQPSAdhocAppFactory();
+
+  AppWrapper CreateApp(GraphDBSession& graph) override;
+};
+
+// Factory
+class HQPSProcedureAppFactory : public AppFactoryBase {
+ public:
+  HQPSProcedureAppFactory();
+  ~HQPSProcedureAppFactory();
+
+  AppWrapper CreateApp(GraphDBSession& graph) override;
+};
+}  // namespace gs
+
+#endif  // ENGINES_GRAPH_DB_APP_HQPS_APP_H_