Reorganize and cleanup proto definitions. (#1547)

alibaba · May 16, 2022 · 0115a72 · 0115a72
1 parent 8fbf192
commit 0115a72
Show file tree

Hide file tree

Showing 50 changed files with 159 additions and 209 deletions.
diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt
@@ -189,12 +189,10 @@ include("cmake/FindGRPC.cmake")
 include_directories(${GRPC_INCLUDE_DIR})
 
 # Generate proto
-execute_process(COMMAND python3 proto/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}/proto" --cpp
-        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/..)
-execute_process(COMMAND python3 proto/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}/../python" --python
+execute_process(COMMAND python3 python/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}" --cpp
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/..)
 
-file(GLOB PROTO_CPP_FILES "proto/graphscope/proto/*.cc")
+file(GLOB PROTO_CPP_FILES "graphscope/proto/*.cc")
 file(GLOB CORE_RPC_SRC_FILES "core/server/*.cc")
 
 # Add targets
@@ -407,7 +405,7 @@ install_gsa_binary(gs_util)
 install_gsa_headers("${PROJECT_SOURCE_DIR}/apps")
 install_gsa_headers("${PROJECT_SOURCE_DIR}/benchmarks")
 install_gsa_headers("${PROJECT_SOURCE_DIR}/core")
-install_gsa_headers("${PROJECT_SOURCE_DIR}/proto")
+install_gsa_headers("${PROJECT_SOURCE_DIR}/graphscope")
 install_gsa_app_frames("${PROJECT_SOURCE_DIR}/frame")
 
 install_gsa_dependency_modules("${PROJECT_SOURCE_DIR}/cmake")

diff --git a/analytical_engine/core/app/app_invoker.h b/analytical_engine/core/app/app_invoker.h
@@ -28,8 +28,8 @@
 #endif
 #include "core/config.h"
 #include "core/error.h"
-#include "proto/graphscope/proto/data_types.pb.h"
-#include "proto/graphscope/proto/query_args.pb.h"
+#include "graphscope/proto/data_types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/error.h b/analytical_engine/core/error.h
@@ -26,7 +26,7 @@
 
 #include "vineyard/graph/utils/error.h"
 
-#include "proto/graphscope/proto/error_codes.pb.h"
+#include "graphscope/proto/error_codes.pb.h"
 #include "utils/mpi_utils.h"
 
 namespace gs {

diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h
@@ -37,7 +37,7 @@
 #include "core/object/dynamic.h"
 #include "core/utils/convert_utils.h"
 #include "core/utils/partitioner.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/fragment/dynamic_projected_fragment.h b/analytical_engine/core/fragment/dynamic_projected_fragment.h
@@ -26,7 +26,7 @@
 
 #include "core/config.h"
 #include "core/fragment/dynamic_fragment.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 namespace dynamic_projected_fragment_impl {

diff --git a/analytical_engine/core/fragment/fragment_reporter.h b/analytical_engine/core/fragment/fragment_reporter.h
@@ -32,7 +32,7 @@
 #include "core/server/rpc_utils.h"
 #include "core/utils/convert_utils.h"
 #include "core/utils/msgpack_utils.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 /**

diff --git a/analytical_engine/core/grape_instance.cc b/analytical_engine/core/grape_instance.cc
@@ -44,7 +44,7 @@
 #include "core/object/projector.h"
 #include "core/server/rpc_utils.h"
 #include "core/utils/fragment_traits.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/grape_instance.h b/analytical_engine/core/grape_instance.h
@@ -39,8 +39,7 @@
 #include "core/server/dispatcher.h"
 #include "core/server/graphscope_service.h"
 #include "core/server/rpc_utils.h"
-#include "proto/graphscope/proto/query_args.pb.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 /**

diff --git a/analytical_engine/core/io/property_parser.h b/analytical_engine/core/io/property_parser.h
@@ -33,9 +33,9 @@
 #include "vineyard/basic/ds/arrow_utils.h"
 
 #include "core/server/rpc_utils.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/op_def.pb.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/op_def.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 template <typename Key, typename Value>
 using ProtobufMap = ::google::protobuf::Map<Key, Value>;

diff --git a/analytical_engine/core/object/app_entry.h b/analytical_engine/core/object/app_entry.h
@@ -28,7 +28,7 @@
 #include "core/error.h"
 #include "core/object/gs_object.h"
 #include "core/utils/lib_utils.h"
-#include "proto/graphscope/proto/query_args.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/object/dynamic.h b/analytical_engine/core/object/dynamic.h
@@ -27,7 +27,7 @@
 #include "rapidjson/writer.h"
 
 #include "grape/serialization/in_archive.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/object/fragment_wrapper.h b/analytical_engine/core/object/fragment_wrapper.h
@@ -45,8 +45,8 @@
 #include "core/object/gs_object.h"
 #include "core/object/i_fragment_wrapper.h"
 #include "core/utils/transform_utils.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/object/graph_utils.h b/analytical_engine/core/object/graph_utils.h
@@ -28,7 +28,7 @@
 #include "core/object/i_fragment_wrapper.h"
 #include "core/server/rpc_utils.h"
 #include "core/utils/lib_utils.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/object/i_fragment_wrapper.h b/analytical_engine/core/object/i_fragment_wrapper.h
@@ -27,8 +27,8 @@
 #include "core/context/i_context.h"
 #include "core/object/gs_object.h"
 #include "core/server/rpc_utils.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
 
 namespace gs {
 class IContextWrapper;

diff --git a/analytical_engine/core/server/command_detail.h b/analytical_engine/core/server/command_detail.h
@@ -23,9 +23,8 @@
 #include "grape/serialization/in_archive.h"
 #include "grape/serialization/out_archive.h"
 
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/query_args.pb.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/server/dispatcher.h b/analytical_engine/core/server/dispatcher.h
@@ -32,7 +32,7 @@
 #include "core/error.h"
 #include "core/server/command_detail.h"
 #include "core/utils/mpi_utils.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
 
 namespace gs {
 

diff --git a/analytical_engine/core/server/graphscope_service.h b/analytical_engine/core/server/graphscope_service.h
@@ -23,11 +23,11 @@
 #include <vector>
 
 #include "core/server/dispatcher.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/engine_service.grpc.pb.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
-#include "proto/graphscope/proto/message.pb.h"
-#include "proto/graphscope/proto/op_def.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/engine_service.grpc.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/message.pb.h"
+#include "graphscope/proto/op_def.pb.h"
 
 #include "boost/lexical_cast.hpp"
 
@@ -71,8 +71,8 @@ class GraphScopeService final : public EngineService::Service {
     const std::string& data = result.data();
 
     // has_large_result
-    op_result->set_has_large_result(result.has_large_data());
-    if (op_result->has_large_result()) {
+    op_result->mutable_meta()->set_has_large_result(result.has_large_data());
+    if (op_result->meta().has_large_result()) {
       // split
       for (size_t i = 0; i < data.size(); i += chunk_size_) {
         RunStepResponse response_body;

diff --git a/analytical_engine/core/server/rpc_utils.h b/analytical_engine/core/server/rpc_utils.h
@@ -28,10 +28,10 @@
 #include "core/config.h"
 #include "core/error.h"
 #include "core/server/command_detail.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/message.pb.h"
-#include "proto/graphscope/proto/op_def.pb.h"
-#include "proto/graphscope/proto/types.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/message.pb.h"
+#include "graphscope/proto/op_def.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 namespace gs {
 namespace rpc {
@@ -75,13 +75,13 @@ inline float get_param_impl<float>(const std::map<int, rpc::AttrValue>& params,
 template <>
 inline rpc::graph::GraphTypePb get_param_impl<rpc::graph::GraphTypePb>(
     const std::map<int, rpc::AttrValue>& params, rpc::ParamKey key) {
-  return params.at(key).graph_type();
+  return static_cast<rpc::graph::GraphTypePb>(params.at(key).i());
 }
 
 template <>
 inline rpc::ModifyType get_param_impl<rpc::ModifyType>(
     const std::map<int, rpc::AttrValue>& params, rpc::ParamKey key) {
-  return params.at(key).modify_type();
+  return static_cast<rpc::ModifyType>(params.at(key).i());
 }
 
 template <>
@@ -93,7 +93,7 @@ inline rpc::AttrValue_ListValue get_param_impl<rpc::AttrValue_ListValue>(
 template <>
 inline rpc::ReportType get_param_impl<rpc::ReportType>(
     const std::map<int, rpc::AttrValue>& params, rpc::ParamKey key) {
-  return params.at(key).report_type();
+  return static_cast<rpc::ReportType>(params.at(key).i());
 }
 
 /**

diff --git a/analytical_engine/frame/app_frame.cc b/analytical_engine/frame/app_frame.cc
@@ -21,7 +21,6 @@
 #include "core/app/app_invoker.h"
 #include "core/error.h"
 #include "frame/ctx_wrapper_builder.h"
-#include "proto/graphscope/proto/query_args.pb.h"
 
 #define DO_QUOTE(X) #X
 #define QUOTE(X) DO_QUOTE(X)

diff --git a/analytical_engine/frame/cython_app_frame.cc b/analytical_engine/frame/cython_app_frame.cc
@@ -30,8 +30,8 @@
 #include "core/app/pregel/pregel_property_app_base.h"
 #include "core/error.h"
 #include "frame/ctx_wrapper_builder.h"
-#include "proto/graphscope/proto/data_types.pb.h"
-#include "proto/graphscope/proto/query_args.pb.h"
+#include "graphscope/proto/data_types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 using string = std::string;
 

diff --git a/analytical_engine/frame/cython_pie_app_frame.cc b/analytical_engine/frame/cython_pie_app_frame.cc
@@ -28,8 +28,8 @@
 #include "core/app/app_invoker.h"
 #include "core/error.h"
 #include "frame/ctx_wrapper_builder.h"
-#include "proto/graphscope/proto/data_types.pb.h"
-#include "proto/graphscope/proto/query_args.pb.h"
+#include "graphscope/proto/data_types.pb.h"
+#include "graphscope/proto/types.pb.h"
 
 using string = std::string;
 

diff --git a/analytical_engine/frame/project_frame.cc b/analytical_engine/frame/project_frame.cc
@@ -28,7 +28,7 @@
 #include "core/object/fragment_wrapper.h"
 #include "core/server/rpc_utils.h"
 #include "core/utils/fragment_traits.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
 
 #if !defined(_PROJECTED_GRAPH_TYPE)
 #error "_PROJECTED_GRAPH_TYPE is undefined"

diff --git a/analytical_engine/frame/property_graph_frame.cc b/analytical_engine/frame/property_graph_frame.cc
@@ -30,8 +30,8 @@
 #include "core/server/rpc_utils.h"
 #include "core/utils/fragment_traits.h"
 #include "core/vertex_map/arrow_projected_vertex_map.h"
-#include "proto/graphscope/proto/attr_value.pb.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
+#include "graphscope/proto/attr_value.pb.h"
+#include "graphscope/proto/graph_def.pb.h"
 
 #if !defined(_GRAPH_TYPE)
 #error Missing _GRAPH_TYPE

diff --git a/analytical_engine/test/app_tests.sh b/analytical_engine/test/app_tests.sh
@@ -181,13 +181,13 @@ function run_vy() {
     do
       for ((dst=0;dst<v_label_num;++dst))
       do
-	if [ "$first" = true ]
+    if [ "$first" = true ]
         then
           first=false
           cmd="${cmd}${e_prefix}_${src}_${dst}_${e}#src_label=v${src}&dst_label=v${dst}&label=e${e}"
         else
           cmd="${cmd};${e_prefix}_${src}_${dst}_${e}#src_label=v${src}&dst_label=v${dst}&label=e${e}"
-	fi
+    fi
       done
     done
     cmd="${cmd}'"

diff --git a/analytical_engine/test/run_java_app.cc b/analytical_engine/test/run_java_app.cc
@@ -49,9 +49,9 @@
 #include "core/loader/arrow_fragment_loader.h"
 #include "core/object/fragment_wrapper.h"
 #include "core/utils/transform_utils.h"
+#include "graphscope/proto/graph_def.pb.h"
 #include "java_pie/java_pie_projected_parallel_app.h"
 #include "java_pie/java_pie_property_parallel_app.h"
-#include "proto/graphscope/proto/graph_def.pb.h"
 
 using FragmentType =
     vineyard::ArrowFragment<vineyard::property_graph_types::OID_TYPE,