From 1145b054acc0cd4863163777e5086247660a387d Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 09:47:42 +0800 Subject: [PATCH 1/8] unify gae, python and groot proto into proto directory --- analytical_engine/CMakeLists.txt | 6 +- analytical_engine/core/app/app_invoker.h | 4 +- .../core/context/vertex_data_context.h | 2 +- analytical_engine/core/error.h | 2 +- .../core/fragment/arrow_projected_fragment.h | 2 +- .../core/fragment/dynamic_fragment.h | 2 +- .../core/fragment/fragment_reporter.h | 2 +- analytical_engine/core/grape_instance.cc | 6 +- analytical_engine/core/grape_instance.h | 2 +- analytical_engine/core/io/property_parser.h | 4 +- analytical_engine/core/object/dynamic.h | 2 +- .../core/object/fragment_wrapper.h | 4 +- .../core/server/command_detail.cc | 4 +- .../core/server/command_detail.h | 4 +- analytical_engine/core/server/dispatcher.cc | 4 +- analytical_engine/core/server/dispatcher.h | 6 +- .../core/server/graphscope_service.cc | 10 +-- .../core/server/graphscope_service.h | 6 +- analytical_engine/core/server/rpc_utils.h | 8 +- analytical_engine/frame/cython_app_frame.cc | 4 +- .../frame/cython_pie_app_frame.cc | 4 +- analytical_engine/frame/project_frame.cc | 2 +- .../frame/property_graph_frame.cc | 4 +- analytical_engine/test/run_java_app.cc | 2 +- .../gscoordinator/kubernetes_launcher.py | 4 +- coordinator/gscoordinator/op_executor.py | 2 +- .../template/CMakeLists.template | 2 +- interactive_engine/common/pom.xml | 2 +- interactive_engine/groot-client/pom.xml | 10 +-- .../proto/ingest_progress_service.proto | 31 -------- .../proto/ingestor_write_service.proto | 35 --------- interactive_engine/sdk-common/pom.xml | 10 +-- proto/__init__.py | 17 ++++ .../proto => proto}/attr_value.proto | 0 .../proto => proto}/coordinator_service.proto | 2 +- .../proto => proto}/data_types.proto | 0 .../proto => proto}/ddl_service.proto | 2 +- .../proto => proto}/engine_service.proto | 2 +- .../proto => proto}/error_codes.proto | 0 .../proto => proto}/graph_def.proto | 0 .../proto => proto/groot}/cluster.proto | 0 .../groot}/frontend_snapshot_service.proto | 0 .../groot}/id_allocate_service.proto | 0 .../groot}/ingestor_snapshot_service.proto | 27 +++++++ .../proto => proto/groot}/meta_service.proto | 0 .../groot}/metrics_collect_service.proto | 0 .../groot}/schema_service.proto | 0 .../groot}/sdk/backup_service.proto | 0 .../proto => proto/groot}/sdk/client.proto | 0 .../groot}/sdk/client_backup_service.proto | 0 .../groot}/sdk/client_ddl_service.proto | 0 .../proto => proto/groot}/sdk/common.proto | 0 .../proto => proto/groot}/sdk/model.proto | 0 .../proto => proto/groot}/sdk/schema.proto | 0 .../groot}/snapshot_commit_service.proto | 0 .../groot}/store_backup_service.proto | 0 .../groot}/store_ingest_service.proto | 0 .../groot}/store_schema_service.proto | 0 .../groot}/store_write_service.proto | 2 +- .../graphscope/proto => proto}/message.proto | 6 +- .../graphscope/proto => proto}/op_def.proto | 8 +- .../proto => proto}/proto_generator.py | 77 +++++++++---------- .../graphscope/proto => proto}/types.proto | 0 .../proto => proto}/write_service.proto | 0 python/graphscope/proto/__init__.py | 5 ++ python/setup.py | 42 +++++----- 66 files changed, 183 insertions(+), 199 deletions(-) delete mode 100644 interactive_engine/proto/ingest_progress_service.proto delete mode 100644 interactive_engine/proto/ingestor_write_service.proto create mode 100644 proto/__init__.py rename {python/graphscope/proto => proto}/attr_value.proto (100%) rename {python/graphscope/proto => proto}/coordinator_service.proto (97%) rename {python/graphscope/proto => proto}/data_types.proto (100%) rename {python/graphscope/proto => proto}/ddl_service.proto (98%) rename {python/graphscope/proto => proto}/engine_service.proto (95%) rename {python/graphscope/proto => proto}/error_codes.proto (100%) rename {python/graphscope/proto => proto}/graph_def.proto (100%) rename {interactive_engine/proto => proto/groot}/cluster.proto (100%) rename {interactive_engine/proto => proto/groot}/frontend_snapshot_service.proto (100%) rename {interactive_engine/proto => proto/groot}/id_allocate_service.proto (100%) rename {interactive_engine/proto => proto/groot}/ingestor_snapshot_service.proto (66%) rename {interactive_engine/proto => proto/groot}/meta_service.proto (100%) rename {interactive_engine/proto => proto/groot}/metrics_collect_service.proto (100%) rename {interactive_engine/proto => proto/groot}/schema_service.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/backup_service.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/client.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/client_backup_service.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/client_ddl_service.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/common.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/model.proto (100%) rename {interactive_engine/proto => proto/groot}/sdk/schema.proto (100%) rename {interactive_engine/proto => proto/groot}/snapshot_commit_service.proto (100%) rename {interactive_engine/proto => proto/groot}/store_backup_service.proto (100%) rename {interactive_engine/proto => proto/groot}/store_ingest_service.proto (100%) rename {interactive_engine/proto => proto/groot}/store_schema_service.proto (100%) rename {interactive_engine/proto => proto/groot}/store_write_service.proto (94%) rename {python/graphscope/proto => proto}/message.proto (97%) rename {python/graphscope/proto => proto}/op_def.proto (92%) rename {python/graphscope/proto => proto}/proto_generator.py (72%) mode change 100755 => 100644 rename {python/graphscope/proto => proto}/types.proto (100%) rename {python/graphscope/proto => proto}/write_service.proto (100%) diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt index 55425acd8d5b..8a39d3ddc5ef 100644 --- a/analytical_engine/CMakeLists.txt +++ b/analytical_engine/CMakeLists.txt @@ -278,10 +278,10 @@ if(Arrow_FOUND AND Arrow_VERSION VERSION_GREATER 9.0.1) endif() # Generate proto -execute_process(COMMAND python3 python/graphscope/proto/proto_generator.py "${PROJECT_SOURCE_DIR}" --cpp - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/..) +execute_process(COMMAND python3 ../proto/proto_generator.py "${PROJECT_SOURCE_DIR}/proto" --cpp + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) -file(GLOB PROTO_CPP_FILES "graphscope/proto/*.cc") +file(GLOB PROTO_CPP_FILES "proto/*.cc") file(GLOB CORE_RPC_SRC_FILES "core/server/*.cc") # Add targets diff --git a/analytical_engine/core/app/app_invoker.h b/analytical_engine/core/app/app_invoker.h index be4c38ff5808..eb3cf8c21942 100644 --- a/analytical_engine/core/app/app_invoker.h +++ b/analytical_engine/core/app/app_invoker.h @@ -29,8 +29,8 @@ #endif #include "core/config.h" #include "core/error.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/context/vertex_data_context.h b/analytical_engine/core/context/vertex_data_context.h index 1ae381d38d3d..c1671853899c 100644 --- a/analytical_engine/core/context/vertex_data_context.h +++ b/analytical_engine/core/context/vertex_data_context.h @@ -50,7 +50,7 @@ #include "core/server/rpc_utils.h" #include "core/utils/mpi_utils.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" #define CONTEXT_TYPE_VERTEX_DATA "vertex_data" #define CONTEXT_TYPE_LABELED_VERTEX_DATA "labeled_vertex_data" diff --git a/analytical_engine/core/error.h b/analytical_engine/core/error.h index e2bfb51dc48b..a5a1e0c6f60f 100644 --- a/analytical_engine/core/error.h +++ b/analytical_engine/core/error.h @@ -24,7 +24,7 @@ #include "vineyard/graph/utils/error.h" // IWYU pragma: export -#include "graphscope/proto/error_codes.pb.h" // IWYU pragma: export +#include "proto/error_codes.pb.h" // IWYU pragma: export namespace gs { diff --git a/analytical_engine/core/fragment/arrow_projected_fragment.h b/analytical_engine/core/fragment/arrow_projected_fragment.h index 8afeee239a7a..8a1e2f368110 100644 --- a/analytical_engine/core/fragment/arrow_projected_fragment.h +++ b/analytical_engine/core/fragment/arrow_projected_fragment.h @@ -41,7 +41,7 @@ #include "core/config.h" #include "core/fragment/arrow_projected_fragment_base.h" // IWYU pragma: export #include "core/vertex_map/arrow_projected_vertex_map.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace arrow { class Array; diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h index fa9e1db8fae5..734e745e2227 100644 --- a/analytical_engine/core/fragment/dynamic_fragment.h +++ b/analytical_engine/core/fragment/dynamic_fragment.h @@ -42,7 +42,7 @@ #include "core/object/dynamic.h" #include "core/utils/convert_utils.h" #include "core/utils/partitioner.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace gs { diff --git a/analytical_engine/core/fragment/fragment_reporter.h b/analytical_engine/core/fragment/fragment_reporter.h index 1ac8d424020f..0cb332b4a721 100644 --- a/analytical_engine/core/fragment/fragment_reporter.h +++ b/analytical_engine/core/fragment/fragment_reporter.h @@ -39,7 +39,7 @@ #include "core/server/rpc_utils.h" #include "core/utils/convert_utils.h" #include "core/utils/msgpack_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/grape_instance.cc b/analytical_engine/core/grape_instance.cc index 97f701931ac5..c49357f73875 100644 --- a/analytical_engine/core/grape_instance.cc +++ b/analytical_engine/core/grape_instance.cc @@ -65,9 +65,9 @@ #include "core/server/command_detail.h" #include "core/server/rpc_utils.h" #include "core/utils/mpi_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/grape_instance.h b/analytical_engine/core/grape_instance.h index 460eaa8ff784..03e32fbb6fa8 100644 --- a/analytical_engine/core/grape_instance.h +++ b/analytical_engine/core/grape_instance.h @@ -38,7 +38,7 @@ #include "core/object/object_manager.h" #include "core/server/dispatcher.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/types.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/io/property_parser.h b/analytical_engine/core/io/property_parser.h index a55e3201e29d..acf3e1da25ca 100644 --- a/analytical_engine/core/io/property_parser.h +++ b/analytical_engine/core/io/property_parser.h @@ -34,8 +34,8 @@ #include "vineyard/common/util/status.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/object/dynamic.h b/analytical_engine/core/object/dynamic.h index 35d7c6c48ef3..d1b6a30fb8f8 100644 --- a/analytical_engine/core/object/dynamic.h +++ b/analytical_engine/core/object/dynamic.h @@ -32,7 +32,7 @@ // IWYU pragma: end_exports #include "grape/serialization/in_archive.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/graph_def.pb.h" namespace gs { diff --git a/analytical_engine/core/object/fragment_wrapper.h b/analytical_engine/core/object/fragment_wrapper.h index a178dc435e87..aded8aaaea68 100644 --- a/analytical_engine/core/object/fragment_wrapper.h +++ b/analytical_engine/core/object/fragment_wrapper.h @@ -61,8 +61,8 @@ #include "core/object/i_fragment_wrapper.h" #include "core/server/rpc_utils.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/command_detail.cc b/analytical_engine/core/server/command_detail.cc index 47e50b51a06a..49a09539f3c1 100644 --- a/analytical_engine/core/server/command_detail.cc +++ b/analytical_engine/core/server/command_detail.cc @@ -21,8 +21,8 @@ #include "grape/serialization/in_archive.h" #include "grape/serialization/out_archive.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace gs { diff --git a/analytical_engine/core/server/command_detail.h b/analytical_engine/core/server/command_detail.h index c33db5e77ae2..efc75ca65f25 100644 --- a/analytical_engine/core/server/command_detail.h +++ b/analytical_engine/core/server/command_detail.h @@ -19,8 +19,8 @@ #include #include -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/types.pb.h" namespace grape { class InArchive; diff --git a/analytical_engine/core/server/dispatcher.cc b/analytical_engine/core/server/dispatcher.cc index e10f4fd74d0b..efa3e535c7d3 100644 --- a/analytical_engine/core/server/dispatcher.cc +++ b/analytical_engine/core/server/dispatcher.cc @@ -37,8 +37,8 @@ #include "core/error.h" #include "core/io/property_parser.h" #include "core/server/command_detail.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/dispatcher.h b/analytical_engine/core/server/dispatcher.h index 1b2a7d30ce47..ecbffda33805 100644 --- a/analytical_engine/core/server/dispatcher.h +++ b/analytical_engine/core/server/dispatcher.h @@ -25,9 +25,9 @@ #include "grape/worker/comm_spec.h" #include "vineyard/common/util/blocking_queue.h" -#include "graphscope/proto/error_codes.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/error_codes.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/core/server/graphscope_service.cc b/analytical_engine/core/server/graphscope_service.cc index 84dd07aa35c8..3e646a8ffae6 100644 --- a/analytical_engine/core/server/graphscope_service.cc +++ b/analytical_engine/core/server/graphscope_service.cc @@ -26,11 +26,11 @@ #include "grpcpp/support/sync_stream.h" #include "core/server/rpc_utils.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/error_codes.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/message.pb.h" -#include "graphscope/proto/op_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/error_codes.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/message.pb.h" +#include "proto/op_def.pb.h" namespace gs { struct CommandDetail; diff --git a/analytical_engine/core/server/graphscope_service.h b/analytical_engine/core/server/graphscope_service.h index 435f3f87c74a..8e5850cab290 100644 --- a/analytical_engine/core/server/graphscope_service.h +++ b/analytical_engine/core/server/graphscope_service.h @@ -29,9 +29,9 @@ #include "grpcpp/support/sync_stream.h" #include "core/server/dispatcher.h" -#include "graphscope/proto/engine_service.grpc.pb.h" -#include "graphscope/proto/message.pb.h" -#include "graphscope/proto/op_def.pb.h" +#include "proto/engine_service.grpc.pb.h" +#include "proto/message.pb.h" +#include "proto/op_def.pb.h" namespace gs { namespace rpc { diff --git a/analytical_engine/core/server/rpc_utils.h b/analytical_engine/core/server/rpc_utils.h index bd7aa6b3b7fd..83314d84f46e 100644 --- a/analytical_engine/core/server/rpc_utils.h +++ b/analytical_engine/core/server/rpc_utils.h @@ -28,10 +28,10 @@ #include "core/config.h" #include "core/server/command_detail.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" -#include "graphscope/proto/op_def.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" +#include "proto/op_def.pb.h" +#include "proto/types.pb.h" namespace bl = boost::leaf; diff --git a/analytical_engine/frame/cython_app_frame.cc b/analytical_engine/frame/cython_app_frame.cc index 61f595bbc398..71914faff1ea 100644 --- a/analytical_engine/frame/cython_app_frame.cc +++ b/analytical_engine/frame/cython_app_frame.cc @@ -47,8 +47,8 @@ #include "core/app/pregel/pregel_property_app_base.h" #include "core/error.h" #include "frame/ctx_wrapper_builder.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" #include QUOTE(_APP_HEADER) diff --git a/analytical_engine/frame/cython_pie_app_frame.cc b/analytical_engine/frame/cython_pie_app_frame.cc index c8084a5b31fe..2771d98eeff5 100644 --- a/analytical_engine/frame/cython_pie_app_frame.cc +++ b/analytical_engine/frame/cython_pie_app_frame.cc @@ -45,8 +45,8 @@ #include "core/app/app_invoker.h" #include "core/error.h" #include "frame/ctx_wrapper_builder.h" -#include "graphscope/proto/data_types.pb.h" -#include "graphscope/proto/types.pb.h" +#include "proto/data_types.pb.h" +#include "proto/types.pb.h" #ifdef _APP_HEADER #include QUOTE(_APP_HEADER) diff --git a/analytical_engine/frame/project_frame.cc b/analytical_engine/frame/project_frame.cc index 112ce3743012..bf156240fff2 100644 --- a/analytical_engine/frame/project_frame.cc +++ b/analytical_engine/frame/project_frame.cc @@ -30,7 +30,7 @@ #include "core/object/fragment_wrapper.h" #include "core/server/rpc_utils.h" #include "core/utils/fragment_traits.h" -#include "graphscope/proto/attr_value.pb.h" +#include "proto/attr_value.pb.h" #if !defined(_PROJECTED_GRAPH_TYPE) #error "_PROJECTED_GRAPH_TYPE is undefined" diff --git a/analytical_engine/frame/property_graph_frame.cc b/analytical_engine/frame/property_graph_frame.cc index 9e0645e676c5..50ac6e9d89e7 100644 --- a/analytical_engine/frame/property_graph_frame.cc +++ b/analytical_engine/frame/property_graph_frame.cc @@ -33,8 +33,8 @@ #include "core/server/rpc_utils.h" #include "core/utils/fragment_traits.h" #include "core/vertex_map/arrow_projected_vertex_map.h" -#include "graphscope/proto/attr_value.pb.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/attr_value.pb.h" +#include "proto/graph_def.pb.h" #if !defined(_GRAPH_TYPE) #error Missing _GRAPH_TYPE diff --git a/analytical_engine/test/run_java_app.cc b/analytical_engine/test/run_java_app.cc index 118dd5a8feb9..d5df99a1bf61 100644 --- a/analytical_engine/test/run_java_app.cc +++ b/analytical_engine/test/run_java_app.cc @@ -49,7 +49,7 @@ #include "core/loader/arrow_fragment_loader.h" #include "core/object/fragment_wrapper.h" #include "core/utils/transform_utils.h" -#include "graphscope/proto/graph_def.pb.h" +#include "proto/graph_def.pb.h" #include "java_pie/java_pie_projected_parallel_app.h" #include "java_pie/java_pie_property_parallel_app.h" diff --git a/coordinator/gscoordinator/kubernetes_launcher.py b/coordinator/gscoordinator/kubernetes_launcher.py index a83a18914dcf..062a7afe7cc7 100644 --- a/coordinator/gscoordinator/kubernetes_launcher.py +++ b/coordinator/gscoordinator/kubernetes_launcher.py @@ -121,6 +121,8 @@ def __init__( self._apps_api = kube_client.AppsV1Api(self._api_client) self._resource_object = ResourceManager(self._api_client) + self._serving = False + self._instance_id = instance_id self._namespace = namespace self._delete_namespace = delete_namespace @@ -1148,7 +1150,7 @@ def _allocate_analytical_engine(self): def _distribute_analytical_process(self, pod_name_list, pod_ip_list): # generate and distribute hostfile - hosts = os.path.join(get_tempdir(), "kube_hosts") + hosts = os.path.join(get_tempdir(), "hosts_of_nodes") with open(hosts, "w") as f: for i, pod_ip in enumerate(pod_ip_list): f.write(f"{pod_ip} {pod_name_list[i]}\n") diff --git a/coordinator/gscoordinator/op_executor.py b/coordinator/gscoordinator/op_executor.py index 59bb3e3e3d88..4c5499ec78b3 100644 --- a/coordinator/gscoordinator/op_executor.py +++ b/coordinator/gscoordinator/op_executor.py @@ -355,7 +355,7 @@ def _create_analytical_grpc_stub(self): time.sleep(delay) delay *= 2 # back off raise RuntimeError( - "Failed to connect to engine in 60s, deployment may failed. Please check coordinator log for details" + "Failed to connect to engine in a reasonable time, deployment may failed. Please check coordinator log for details" ) @property diff --git a/coordinator/gscoordinator/template/CMakeLists.template b/coordinator/gscoordinator/template/CMakeLists.template index 2be48666c9ce..17c6cfdddba7 100644 --- a/coordinator/gscoordinator/template/CMakeLists.template +++ b/coordinator/gscoordinator/template/CMakeLists.template @@ -263,7 +263,7 @@ endif() if(GRAPHSCOPE_ANALYTICAL_HOME) include_directories("${GRAPHSCOPE_ANALYTICAL_INCLUDE_DIRS}") include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope/apps") - include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope/proto") + include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/proto") # include vineyard---------------------------------------------------------- include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/vineyard") else() diff --git a/interactive_engine/common/pom.xml b/interactive_engine/common/pom.xml index bbd719967862..58ed3c8b109a 100644 --- a/interactive_engine/common/pom.xml +++ b/interactive_engine/common/pom.xml @@ -84,7 +84,7 @@ interactive - ../proto + ../../proto/groot/ sdk/* diff --git a/interactive_engine/groot-client/pom.xml b/interactive_engine/groot-client/pom.xml index 7b1baf1d36fb..7bb85f11de01 100644 --- a/interactive_engine/groot-client/pom.xml +++ b/interactive_engine/groot-client/pom.xml @@ -90,7 +90,7 @@ sdk - ../proto + ../../proto/groot sdk/* @@ -103,11 +103,11 @@ graphscope - ../../python + ../../proto - graphscope/proto/graph_def.proto - graphscope/proto/ddl_service.proto - graphscope/proto/write_service.proto + graph_def.proto + ddl_service.proto + write_service.proto diff --git a/interactive_engine/proto/ingest_progress_service.proto b/interactive_engine/proto/ingest_progress_service.proto deleted file mode 100644 index baefde0644c1..000000000000 --- a/interactive_engine/proto/ingest_progress_service.proto +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "com.alibaba.graphscope.proto.groot"; -option java_multiple_files = true; - -service IngestProgress { - rpc getTailOffsets(GetTailOffsetsRequest) returns(GetTailOffsetsResponse); -} - -message GetTailOffsetsRequest { - repeated int32 queueId = 1; -} - -message GetTailOffsetsResponse { - repeated int64 offsets = 1; -} diff --git a/interactive_engine/proto/ingestor_write_service.proto b/interactive_engine/proto/ingestor_write_service.proto deleted file mode 100644 index f10bfff8e103..000000000000 --- a/interactive_engine/proto/ingestor_write_service.proto +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -import "sdk/model.proto"; - -option java_package = "com.alibaba.graphscope.proto.groot"; -option java_multiple_files = true; - -service IngestorWrite { - rpc writeIngestor(WriteIngestorRequest) returns (WriteIngestorResponse); -} - -message WriteIngestorRequest { - string requestId = 1; - int32 queueId = 2; - OperationBatchPb operationBatch = 3; -} - -message WriteIngestorResponse { - int64 snapshotId = 1; -} diff --git a/interactive_engine/sdk-common/pom.xml b/interactive_engine/sdk-common/pom.xml index 52a04ac79e07..82d38cbc6edb 100644 --- a/interactive_engine/sdk-common/pom.xml +++ b/interactive_engine/sdk-common/pom.xml @@ -107,7 +107,7 @@ sdk - ../proto + ../../proto/groot sdk/* @@ -120,11 +120,11 @@ graphscope - ../../python + ../../proto - graphscope/proto/graph_def.proto - graphscope/proto/ddl_service.proto - graphscope/proto/write_service.proto + graph_def.proto + ddl_service.proto + write_service.proto diff --git a/proto/__init__.py b/proto/__init__.py new file mode 100644 index 000000000000..618dda69a7be --- /dev/null +++ b/proto/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/graphscope/proto/attr_value.proto b/proto/attr_value.proto similarity index 100% rename from python/graphscope/proto/attr_value.proto rename to proto/attr_value.proto diff --git a/python/graphscope/proto/coordinator_service.proto b/proto/coordinator_service.proto similarity index 97% rename from python/graphscope/proto/coordinator_service.proto rename to proto/coordinator_service.proto index cfe917bed62c..038709bd66ff 100644 --- a/python/graphscope/proto/coordinator_service.proto +++ b/proto/coordinator_service.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/message.proto"; +import "message.proto"; service CoordinatorService { // Connect a session. diff --git a/python/graphscope/proto/data_types.proto b/proto/data_types.proto similarity index 100% rename from python/graphscope/proto/data_types.proto rename to proto/data_types.proto diff --git a/python/graphscope/proto/ddl_service.proto b/proto/ddl_service.proto similarity index 98% rename from python/graphscope/proto/ddl_service.proto rename to proto/ddl_service.proto index af0d49f96a94..e553924bd858 100644 --- a/python/graphscope/proto/ddl_service.proto +++ b/proto/ddl_service.proto @@ -15,7 +15,7 @@ syntax = "proto3"; package gs.rpc.ddl_service.v1; -import "graphscope/proto/graph_def.proto"; +import "graph_def.proto"; option java_package = "com.alibaba.graphscope.proto.ddl"; option java_multiple_files = true; diff --git a/python/graphscope/proto/engine_service.proto b/proto/engine_service.proto similarity index 95% rename from python/graphscope/proto/engine_service.proto rename to proto/engine_service.proto index 76596a826595..da15dbcf86d2 100644 --- a/python/graphscope/proto/engine_service.proto +++ b/proto/engine_service.proto @@ -17,7 +17,7 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/message.proto"; +import "message.proto"; service EngineService { // Drives the graph computation. diff --git a/python/graphscope/proto/error_codes.proto b/proto/error_codes.proto similarity index 100% rename from python/graphscope/proto/error_codes.proto rename to proto/error_codes.proto diff --git a/python/graphscope/proto/graph_def.proto b/proto/graph_def.proto similarity index 100% rename from python/graphscope/proto/graph_def.proto rename to proto/graph_def.proto diff --git a/interactive_engine/proto/cluster.proto b/proto/groot/cluster.proto similarity index 100% rename from interactive_engine/proto/cluster.proto rename to proto/groot/cluster.proto diff --git a/interactive_engine/proto/frontend_snapshot_service.proto b/proto/groot/frontend_snapshot_service.proto similarity index 100% rename from interactive_engine/proto/frontend_snapshot_service.proto rename to proto/groot/frontend_snapshot_service.proto diff --git a/interactive_engine/proto/id_allocate_service.proto b/proto/groot/id_allocate_service.proto similarity index 100% rename from interactive_engine/proto/id_allocate_service.proto rename to proto/groot/id_allocate_service.proto diff --git a/interactive_engine/proto/ingestor_snapshot_service.proto b/proto/groot/ingestor_snapshot_service.proto similarity index 66% rename from interactive_engine/proto/ingestor_snapshot_service.proto rename to proto/groot/ingestor_snapshot_service.proto index c17196f6db66..40898c75a3d0 100644 --- a/interactive_engine/proto/ingestor_snapshot_service.proto +++ b/proto/groot/ingestor_snapshot_service.proto @@ -29,3 +29,30 @@ message AdvanceIngestSnapshotIdRequest { message AdvanceIngestSnapshotIdResponse { int64 previousSnapshotId = 1; } + + +service IngestProgress { + rpc getTailOffsets(GetTailOffsetsRequest) returns(GetTailOffsetsResponse); +} + +message GetTailOffsetsRequest { + repeated int32 queueId = 1; +} + +message GetTailOffsetsResponse { + repeated int64 offsets = 1; +} + +service IngestorWrite { + rpc writeIngestor(WriteIngestorRequest) returns (WriteIngestorResponse); +} + +message WriteIngestorRequest { + string requestId = 1; + int32 queueId = 2; + OperationBatchPb operationBatch = 3; +} + +message WriteIngestorResponse { + int64 snapshotId = 1; +} \ No newline at end of file diff --git a/interactive_engine/proto/meta_service.proto b/proto/groot/meta_service.proto similarity index 100% rename from interactive_engine/proto/meta_service.proto rename to proto/groot/meta_service.proto diff --git a/interactive_engine/proto/metrics_collect_service.proto b/proto/groot/metrics_collect_service.proto similarity index 100% rename from interactive_engine/proto/metrics_collect_service.proto rename to proto/groot/metrics_collect_service.proto diff --git a/interactive_engine/proto/schema_service.proto b/proto/groot/schema_service.proto similarity index 100% rename from interactive_engine/proto/schema_service.proto rename to proto/groot/schema_service.proto diff --git a/interactive_engine/proto/sdk/backup_service.proto b/proto/groot/sdk/backup_service.proto similarity index 100% rename from interactive_engine/proto/sdk/backup_service.proto rename to proto/groot/sdk/backup_service.proto diff --git a/interactive_engine/proto/sdk/client.proto b/proto/groot/sdk/client.proto similarity index 100% rename from interactive_engine/proto/sdk/client.proto rename to proto/groot/sdk/client.proto diff --git a/interactive_engine/proto/sdk/client_backup_service.proto b/proto/groot/sdk/client_backup_service.proto similarity index 100% rename from interactive_engine/proto/sdk/client_backup_service.proto rename to proto/groot/sdk/client_backup_service.proto diff --git a/interactive_engine/proto/sdk/client_ddl_service.proto b/proto/groot/sdk/client_ddl_service.proto similarity index 100% rename from interactive_engine/proto/sdk/client_ddl_service.proto rename to proto/groot/sdk/client_ddl_service.proto diff --git a/interactive_engine/proto/sdk/common.proto b/proto/groot/sdk/common.proto similarity index 100% rename from interactive_engine/proto/sdk/common.proto rename to proto/groot/sdk/common.proto diff --git a/interactive_engine/proto/sdk/model.proto b/proto/groot/sdk/model.proto similarity index 100% rename from interactive_engine/proto/sdk/model.proto rename to proto/groot/sdk/model.proto diff --git a/interactive_engine/proto/sdk/schema.proto b/proto/groot/sdk/schema.proto similarity index 100% rename from interactive_engine/proto/sdk/schema.proto rename to proto/groot/sdk/schema.proto diff --git a/interactive_engine/proto/snapshot_commit_service.proto b/proto/groot/snapshot_commit_service.proto similarity index 100% rename from interactive_engine/proto/snapshot_commit_service.proto rename to proto/groot/snapshot_commit_service.proto diff --git a/interactive_engine/proto/store_backup_service.proto b/proto/groot/store_backup_service.proto similarity index 100% rename from interactive_engine/proto/store_backup_service.proto rename to proto/groot/store_backup_service.proto diff --git a/interactive_engine/proto/store_ingest_service.proto b/proto/groot/store_ingest_service.proto similarity index 100% rename from interactive_engine/proto/store_ingest_service.proto rename to proto/groot/store_ingest_service.proto diff --git a/interactive_engine/proto/store_schema_service.proto b/proto/groot/store_schema_service.proto similarity index 100% rename from interactive_engine/proto/store_schema_service.proto rename to proto/groot/store_schema_service.proto diff --git a/interactive_engine/proto/store_write_service.proto b/proto/groot/store_write_service.proto similarity index 94% rename from interactive_engine/proto/store_write_service.proto rename to proto/groot/store_write_service.proto index f2b4fd7e5b95..35403de8125c 100644 --- a/interactive_engine/proto/store_write_service.proto +++ b/proto/groot/store_write_service.proto @@ -21,7 +21,7 @@ option java_multiple_files = true; import "sdk/model.proto"; service StoreWrite { - rpc writeStore(WriteStoreRequest) returns(WriteStoreResponse); + rpc writeStore(WriteStoreRequest) returns (WriteStoreResponse); } message PartitionToBatchPb { diff --git a/python/graphscope/proto/message.proto b/proto/message.proto similarity index 97% rename from python/graphscope/proto/message.proto rename to proto/message.proto index 3509d7aead60..a6cbff9abf02 100644 --- a/python/graphscope/proto/message.proto +++ b/proto/message.proto @@ -17,9 +17,9 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/error_codes.proto"; -import "graphscope/proto/op_def.proto"; -import "graphscope/proto/types.proto"; +import "error_codes.proto"; +import "op_def.proto"; +import "types.proto"; //////////////////////////////////////////////////////////////////////////////// diff --git a/python/graphscope/proto/op_def.proto b/proto/op_def.proto similarity index 92% rename from python/graphscope/proto/op_def.proto rename to proto/op_def.proto index 036329a07b79..1cd9c6b98d8b 100644 --- a/python/graphscope/proto/op_def.proto +++ b/proto/op_def.proto @@ -17,10 +17,10 @@ syntax = "proto3"; package gs.rpc; -import "graphscope/proto/attr_value.proto"; -import "graphscope/proto/error_codes.proto"; -import "graphscope/proto/graph_def.proto"; -import "graphscope/proto/types.proto"; +import "attr_value.proto"; +import "error_codes.proto"; +import "graph_def.proto"; +import "types.proto"; message OpDef { // Unique key for every OpDef. Usually generated by analytical engine. diff --git a/python/graphscope/proto/proto_generator.py b/proto/proto_generator.py old mode 100755 new mode 100644 similarity index 72% rename from python/graphscope/proto/proto_generator.py rename to proto/proto_generator.py index 503dba2fab5a..87b41630ea24 --- a/python/graphscope/proto/proto_generator.py +++ b/proto/proto_generator.py @@ -24,28 +24,27 @@ def gather_all_proto(proto_dir, suffix="*.proto"): - directory = os.path.join(proto_dir, suffix) - files = glob.glob(directory) - return files + pattern = os.path.join(proto_dir, suffix) + return glob.glob(pattern) def create_path(path): """Utility function to create a path.""" - if os.path.isdir(path): - return - os.makedirs(path, exist_ok=True) + if not os.path.isdir(path): + os.makedirs(path, exist_ok=True) def cpp_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: - subprocess.check_call( - [ + cmd = [ shutil.which("protoc"), - "-I%s" % ".", - "--cpp_out=%s" % output_dir, + "-I.", + f"--cpp_out={output_dir}", proto_file, - ], + ] + subprocess.check_call( + cmd, stderr=subprocess.STDOUT, ) @@ -53,16 +52,17 @@ def cpp_out(relative_dir, output_dir): def python_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: - subprocess.check_call( - [ + cmd=[ sys.executable, "-m", "grpc_tools.protoc", - "-I%s" % ".", - "--python_out=%s" % os.path.join(output_dir), - "--mypy_out=%s" % os.path.join(output_dir), + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", proto_file, - ], + ] + subprocess.check_call( + cmd, stderr=subprocess.STDOUT, ) @@ -74,14 +74,15 @@ def cpp_service_out(relative_dir, output_dir): suffix = "*_service.proto" files = gather_all_proto(relative_dir, suffix) for proto_file in files: - subprocess.check_call( - [ + cmd = [ shutil.which("protoc"), - "-I%s" % ".", - "--grpc_out=%s" % output_dir, - "--plugin=protoc-gen-grpc=%s" % plugin_path, + "-I.", + f"--grpc_out={output_dir}", + f"--plugin=protoc-gen-grpc={plugin_path}", proto_file, - ], + ] + subprocess.check_call( + cmd, stderr=subprocess.STDOUT, ) @@ -90,18 +91,19 @@ def python_service_out(relative_dir, output_dir): suffix = "*_service.proto" files = gather_all_proto(relative_dir, suffix) for proto_file in files: - subprocess.check_call( - [ + cmd = [ sys.executable, "-m", "grpc_tools.protoc", - "-I%s" % ".", - "--python_out=%s" % output_dir, - "--mypy_out=%s" % os.path.join(output_dir), - "--grpc_python_out=%s" % output_dir, - "--mypy_grpc_out=%s" % os.path.join(output_dir), + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", + f"--grpc_python_out={output_dir}", + f"--mypy_grpc_out={output_dir}", proto_file, - ], + ] + subprocess.check_call( + cmd, stderr=subprocess.STDOUT, ) @@ -111,19 +113,16 @@ def python_service_out(relative_dir, output_dir): print("Usage: python proto_generator.py [--cpp] [--python]") sys.exit(1) - # path to 'GraphScope/python/graphscope/proto' - current_dir = os.path.dirname(os.path.abspath(__file__)) - - # path to 'GraphScope/python' - base_dir = os.path.join(current_dir, "../", "../") - os.chdir(base_dir) - output_dir = sys.argv[1] output_dir = os.path.realpath(os.path.realpath(output_dir)) create_path(output_dir) + # path to 'GraphScope/proto' + current_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(current_dir) + # must use relative path - relative_dir = os.path.join(".", "graphscope", "proto") + relative_dir = "." if len(sys.argv) <= 2 or len(sys.argv) > 2 and sys.argv[2] == "--cpp": print("Generating cpp proto to: " + output_dir) cpp_out(relative_dir, output_dir) diff --git a/python/graphscope/proto/types.proto b/proto/types.proto similarity index 100% rename from python/graphscope/proto/types.proto rename to proto/types.proto diff --git a/python/graphscope/proto/write_service.proto b/proto/write_service.proto similarity index 100% rename from python/graphscope/proto/write_service.proto rename to proto/write_service.proto diff --git a/python/graphscope/proto/__init__.py b/python/graphscope/proto/__init__.py index 618dda69a7be..e61cf97316fe 100644 --- a/python/graphscope/proto/__init__.py +++ b/python/graphscope/proto/__init__.py @@ -15,3 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/python/setup.py b/python/setup.py index 50d3c9cdc82b..eba09ce92310 100644 --- a/python/setup.py +++ b/python/setup.py @@ -36,7 +36,7 @@ # https://github.com/pypa/pip/issues/7953 site.ENABLE_USER_SITE = "--user" in sys.argv[1:] -repo_root = os.path.dirname(os.path.abspath(__file__)) +pkg_root = os.path.dirname(os.path.abspath(__file__)) if platform.system() == "Darwin": # see also: https://github.com/python/cpython/issues/100420 @@ -57,20 +57,20 @@ def finalize_options(self): pass def run(self): - subprocess.check_call( - [ + cmd = [ sys.executable, os.path.join( - repo_root, + pkg_root, "..", - "python", - "graphscope", "proto", "proto_generator.py", ), - repo_root, + os.path.join(pkg_root, "graphscope", "proto"), "--python", - ], + ] + print(" ".join(cmd)) + subprocess.check_call( + cmd, env=os.environ.copy(), ) @@ -92,17 +92,17 @@ def finalize_options(self): def run(self): if self.inplace: - subprocess.check_call([sys.executable, "-m", "isort", "."], cwd=repo_root) - subprocess.check_call([sys.executable, "-m", "black", "."], cwd=repo_root) - subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=repo_root) + subprocess.check_call([sys.executable, "-m", "isort", "."], cwd=pkg_root) + subprocess.check_call([sys.executable, "-m", "black", "."], cwd=pkg_root) + subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=pkg_root) else: subprocess.check_call( - [sys.executable, "-m", "isort", "--check", "--diff", "."], cwd=repo_root + [sys.executable, "-m", "isort", "--check", "--diff", "."], cwd=pkg_root ) subprocess.check_call( - [sys.executable, "-m", "black", "--check", "--diff", "."], cwd=repo_root + [sys.executable, "-m", "black", "--check", "--diff", "."], cwd=pkg_root ) - subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=repo_root) + subprocess.check_call([sys.executable, "-m", "flake8", "."], cwd=pkg_root) class CustomBuildPy(build_py): @@ -141,7 +141,7 @@ def run(self): graphlearn_shared_lib = "libgraphlearn_shared.so" if not os.path.isfile( os.path.join( - repo_root, + pkg_root, "..", "learning_engine", "graph-learn", @@ -156,18 +156,18 @@ def run(self): bdist_wheel.run(self) -with open(os.path.join(repo_root, "..", "README.md"), "r", encoding="utf-8") as fp: +with open(os.path.join(pkg_root, "..", "README.md"), "r", encoding="utf-8") as fp: long_description = fp.read() def parsed_reqs(): - with open(os.path.join(repo_root, "requirements.txt"), "r", encoding="utf-8") as fp: + with open(os.path.join(pkg_root, "requirements.txt"), "r", encoding="utf-8") as fp: return fp.read().splitlines() def parsed_dev_reqs(): with open( - os.path.join(repo_root, "requirements-dev.txt"), "r", encoding="utf-8" + os.path.join(pkg_root, "requirements-dev.txt"), "r", encoding="utf-8" ) as fp: return fp.read().splitlines() @@ -202,7 +202,7 @@ def build_learning_engine(): import numpy ROOT_PATH = os.path.abspath( - os.path.join(repo_root, "..", "learning_engine", "graph-learn") + os.path.join(pkg_root, "..", "learning_engine", "graph-learn") ) include_dirs = [] @@ -270,7 +270,7 @@ def parse_version(root, **kwargs): from setuptools_scm.git import parse from setuptools_scm.version import meta - version_file = os.path.join(repo_root, "..", "VERSION") + version_file = os.path.join(pkg_root, "..", "VERSION") if os.path.isfile(version_file): with open(version_file, "r", encoding="utf-8") as fp: return meta(fp.read().strip()) @@ -305,7 +305,7 @@ def parse_version(root, **kwargs): ], keywords="Graph, Large-Scale, Distributed Computing", use_scm_version={ - "root": repo_root, + "root": pkg_root, "parse": parse_version, }, setup_requires=[ From 44970639288ae9cb28624831366a1666b40f47c4 Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 09:52:42 +0800 Subject: [PATCH 2/8] update --- interactive_engine/executor/store/groot/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/executor/store/groot/build.rs b/interactive_engine/executor/store/groot/build.rs index c7217b6d4bc6..37b6fee8aff0 100644 --- a/interactive_engine/executor/store/groot/build.rs +++ b/interactive_engine/executor/store/groot/build.rs @@ -1,7 +1,7 @@ extern crate protoc_grpcio; fn main() { - let proto_root = "../../../proto"; + let proto_root = "../../../../proto/groot"; protoc_grpcio::compile_grpc_protos( &[ proto_root.to_owned() + "/sdk/model.proto", From 2b41995eb3c968fe21441082f59579b0ee529cff Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 10:01:43 +0800 Subject: [PATCH 3/8] update --- proto/groot/ingestor_snapshot_service.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/groot/ingestor_snapshot_service.proto b/proto/groot/ingestor_snapshot_service.proto index 40898c75a3d0..6d9258dd7fb7 100644 --- a/proto/groot/ingestor_snapshot_service.proto +++ b/proto/groot/ingestor_snapshot_service.proto @@ -15,6 +15,8 @@ */ syntax = "proto3"; +import "sdk/model.proto"; + option java_package = "com.alibaba.graphscope.proto.groot"; option java_multiple_files = true; From 86e394a2f32acc7237bfb245e46ac40f53c565f4 Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 10:35:33 +0800 Subject: [PATCH 4/8] update lgraph --- interactive_engine/lgraph/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/lgraph/CMakeLists.txt b/interactive_engine/lgraph/CMakeLists.txt index 241dd3e6f2a9..f43c84d0dbdc 100644 --- a/interactive_engine/lgraph/CMakeLists.txt +++ b/interactive_engine/lgraph/CMakeLists.txt @@ -39,7 +39,7 @@ find_package(gRPC 1.26.0 REQUIRED) find_package(CppKafka 0.3.1 REQUIRED) # Proto file generating -set(PROTO_FILE_DIR ${Lgraph_SOURCE_DIR}/../proto) +set(PROTO_FILE_DIR ${Lgraph_SOURCE_DIR}/../../proto/groot) set(GRPC_FILES ${PROTO_FILE_DIR}/sdk/client.proto ${PROTO_FILE_DIR}/sdk/client_backup_service.proto) From be07dfc4c8f45625ab84f1629fc50bee80d5431e Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 11:06:57 +0800 Subject: [PATCH 5/8] fix gae format --- analytical_engine/test/run_java_app.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytical_engine/test/run_java_app.cc b/analytical_engine/test/run_java_app.cc index d5df99a1bf61..d5b75037a92a 100644 --- a/analytical_engine/test/run_java_app.cc +++ b/analytical_engine/test/run_java_app.cc @@ -49,9 +49,9 @@ #include "core/loader/arrow_fragment_loader.h" #include "core/object/fragment_wrapper.h" #include "core/utils/transform_utils.h" -#include "proto/graph_def.pb.h" #include "java_pie/java_pie_projected_parallel_app.h" #include "java_pie/java_pie_property_parallel_app.h" +#include "proto/graph_def.pb.h" namespace bl = boost::leaf; From 8ac9a4d59dc20df978316a26f1f905f2d4911da9 Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 12:46:59 +0800 Subject: [PATCH 6/8] python format --- proto/__init__.py | 17 ------------ proto/proto_generator.py | 60 ++++++++++++++++++++-------------------- python/setup.py | 20 +++++++------- 3 files changed, 40 insertions(+), 57 deletions(-) delete mode 100644 proto/__init__.py diff --git a/proto/__init__.py b/proto/__init__.py deleted file mode 100644 index 618dda69a7be..000000000000 --- a/proto/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/proto/proto_generator.py b/proto/proto_generator.py index 87b41630ea24..e6d3e06d5274 100644 --- a/proto/proto_generator.py +++ b/proto/proto_generator.py @@ -38,11 +38,11 @@ def cpp_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: cmd = [ - shutil.which("protoc"), - "-I.", - f"--cpp_out={output_dir}", - proto_file, - ] + shutil.which("protoc"), + "-I.", + f"--cpp_out={output_dir}", + proto_file, + ] subprocess.check_call( cmd, stderr=subprocess.STDOUT, @@ -52,15 +52,15 @@ def cpp_out(relative_dir, output_dir): def python_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) for proto_file in files: - cmd=[ - sys.executable, - "-m", - "grpc_tools.protoc", - "-I.", - f"--python_out={output_dir}", - f"--mypy_out={output_dir}", - proto_file, - ] + cmd = [ + sys.executable, + "-m", + "grpc_tools.protoc", + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", + proto_file, + ] subprocess.check_call( cmd, stderr=subprocess.STDOUT, @@ -75,12 +75,12 @@ def cpp_service_out(relative_dir, output_dir): files = gather_all_proto(relative_dir, suffix) for proto_file in files: cmd = [ - shutil.which("protoc"), - "-I.", - f"--grpc_out={output_dir}", - f"--plugin=protoc-gen-grpc={plugin_path}", - proto_file, - ] + shutil.which("protoc"), + "-I.", + f"--grpc_out={output_dir}", + f"--plugin=protoc-gen-grpc={plugin_path}", + proto_file, + ] subprocess.check_call( cmd, stderr=subprocess.STDOUT, @@ -92,16 +92,16 @@ def python_service_out(relative_dir, output_dir): files = gather_all_proto(relative_dir, suffix) for proto_file in files: cmd = [ - sys.executable, - "-m", - "grpc_tools.protoc", - "-I.", - f"--python_out={output_dir}", - f"--mypy_out={output_dir}", - f"--grpc_python_out={output_dir}", - f"--mypy_grpc_out={output_dir}", - proto_file, - ] + sys.executable, + "-m", + "grpc_tools.protoc", + "-I.", + f"--python_out={output_dir}", + f"--mypy_out={output_dir}", + f"--grpc_python_out={output_dir}", + f"--mypy_grpc_out={output_dir}", + proto_file, + ] subprocess.check_call( cmd, stderr=subprocess.STDOUT, diff --git a/python/setup.py b/python/setup.py index eba09ce92310..8eab42409929 100644 --- a/python/setup.py +++ b/python/setup.py @@ -58,16 +58,16 @@ def finalize_options(self): def run(self): cmd = [ - sys.executable, - os.path.join( - pkg_root, - "..", - "proto", - "proto_generator.py", - ), - os.path.join(pkg_root, "graphscope", "proto"), - "--python", - ] + sys.executable, + os.path.join( + pkg_root, + "..", + "proto", + "proto_generator.py", + ), + os.path.join(pkg_root, "graphscope", "proto"), + "--python", + ] print(" ".join(cmd)) subprocess.check_call( cmd, From eb6de83ff971734c4c5725755fb061daf919ecda Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 13:11:56 +0800 Subject: [PATCH 7/8] update --- analytical_engine/CMakeLists.txt | 2 +- coordinator/gscoordinator/template/CMakeLists.template | 2 +- interactive_engine/lgraph/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt index 8a39d3ddc5ef..2913b5473076 100644 --- a/analytical_engine/CMakeLists.txt +++ b/analytical_engine/CMakeLists.txt @@ -569,7 +569,7 @@ endif() install_gsa_headers("${PROJECT_SOURCE_DIR}/apps") install_gsa_headers("${PROJECT_SOURCE_DIR}/benchmarks") install_gsa_headers("${PROJECT_SOURCE_DIR}/core") -install_gsa_headers("${PROJECT_SOURCE_DIR}/graphscope") +install_gsa_headers("${PROJECT_SOURCE_DIR}/proto") install_gsa_app_frames("${PROJECT_SOURCE_DIR}/frame") install_gsa_dependency_modules("${PROJECT_SOURCE_DIR}/cmake") diff --git a/coordinator/gscoordinator/template/CMakeLists.template b/coordinator/gscoordinator/template/CMakeLists.template index 17c6cfdddba7..610d79472a16 100644 --- a/coordinator/gscoordinator/template/CMakeLists.template +++ b/coordinator/gscoordinator/template/CMakeLists.template @@ -263,7 +263,7 @@ endif() if(GRAPHSCOPE_ANALYTICAL_HOME) include_directories("${GRAPHSCOPE_ANALYTICAL_INCLUDE_DIRS}") include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope/apps") - include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/proto") + include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope") # include vineyard---------------------------------------------------------- include_directories("${GRAPHSCOPE_ANALYTICAL_HOME}/include/vineyard") else() diff --git a/interactive_engine/lgraph/CMakeLists.txt b/interactive_engine/lgraph/CMakeLists.txt index f43c84d0dbdc..3560f9bb72e7 100644 --- a/interactive_engine/lgraph/CMakeLists.txt +++ b/interactive_engine/lgraph/CMakeLists.txt @@ -47,7 +47,7 @@ set(PROTO_FILES ${GRPC_FILES} ${PROTO_FILE_DIR}/sdk/model.proto ${PROTO_FILE_DIR}/sdk/common.proto) -set(PROTOBUF_IMPORT_DIRS ${Lgraph_SOURCE_DIR}/../proto) +set(PROTOBUF_IMPORT_DIRS ${Lgraph_SOURCE_DIR}/../../proto/groot) set(PROTO_BUILD_DIR ${Lgraph_BINARY_DIR}/generated) set(PROTO_SRCS_DEST_DIR ${Lgraph_SOURCE_DIR}/src/proto) set(PROTO_HDRS_DEST_DIR ${Lgraph_SOURCE_DIR}/include/lgraph/proto) From a3dd6a3270ccae23baa433b577afc3df863886ab Mon Sep 17 00:00:00 2001 From: siyuan0322 Date: Tue, 18 Jul 2023 15:26:34 +0800 Subject: [PATCH 8/8] update --- coordinator/gscoordinator/kubernetes_launcher.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/coordinator/gscoordinator/kubernetes_launcher.py b/coordinator/gscoordinator/kubernetes_launcher.py index 062a7afe7cc7..ebf0a0d225b6 100644 --- a/coordinator/gscoordinator/kubernetes_launcher.py +++ b/coordinator/gscoordinator/kubernetes_launcher.py @@ -121,8 +121,6 @@ def __init__( self._apps_api = kube_client.AppsV1Api(self._api_client) self._resource_object = ResourceManager(self._api_client) - self._serving = False - self._instance_id = instance_id self._namespace = namespace self._delete_namespace = delete_namespace