-
Notifications
You must be signed in to change notification settings - Fork 5.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add brpc serialization support. #11430
Changes from 47 commits
2acc420
19cb300
b5eb369
68cd387
b3a6973
9cd6f21
b436597
fa0158a
10ad682
8e079b2
ddee50f
a52adb8
3123576
166a79f
18e1743
8888078
5032b93
baf17e2
1bf3783
4544329
8cc2fbe
d7023bb
8c51dd1
f806963
2fe5b68
0355764
8780c08
f8662a1
20bef4f
cb31718
1f49a32
1c5fe25
c7dba8e
6c84d2c
18b0274
a6c7ae3
efe8abb
b1c8a6d
41f62a4
4ccd5b1
3e4fada
a77340d
66e2d44
d2cd653
12ab36d
f8f0fe0
3bb4c13
c1ebb7c
700c15b
0680a27
b2a5c94
b0fc28f
1f97756
1ee917e
aad877a
a1a3899
ee3c7dd
9965ac2
f26322a
0c2a211
aa8a03a
6e109ac
d5c5ec0
f079aea
65397b3
1fb0793
a1a59db
30d9b11
8f7904f
8aa933c
3834fa3
b27585f
c52b365
d4665bb
6fa5ac6
22140d8
e57d380
b77ee81
397b453
c690f15
56a6a15
6b638e0
0a4e750
e9adf66
839ef46
633718a
a4d489b
dc07543
36df67d
0be5ca9
949389f
03476da
0154434
830079b
9afdf93
cb087e4
a096876
8a5068a
3309437
fef1097
f5deaec
c391d49
af74a19
cc46b3c
e3ddd9a
92fb011
d260d5a
1fd988e
eecab31
3585308
046c360
4a06e90
eda4e36
2dd12aa
f6bd695
a76dc4e
a7c7bf2
bc082c7
13ca9c0
7a4adad
a5534e1
c916534
c559cd9
5a93241
65c7d83
f6b4015
db22ddc
88acffc
89394e2
b5b1f90
d492699
d6278a3
e86f86b
8c58ede
f459d4a
842747a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
|
||
IF(MOBILE_INFERENCE OR NOT WITH_DISTRIBUTE) | ||
return() | ||
ENDIF() | ||
|
||
include (ExternalProject) | ||
|
||
SET(OPENSSL_SOURCES_DIR ${THIRD_PARTY_PATH}/openssl) | ||
SET(OPENSSL_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openssl) | ||
SET(OPENSSL_INCLUDE_DIR "${OPENSSL_INSTALL_DIR}/include/" CACHE PATH "openssl include directory." FORCE) | ||
SET(OPENSSL_LIBRARIES "${OPENSSL_INSTALL_DIR}/lib/libssl.a" CACHE FILEPATH "ssl library." FORCE) | ||
SET(CRYPTO_LIBRARIES "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a" CACHE FILEPATH "ssl library." FORCE) | ||
|
||
ExternalProject_Add( | ||
extern_openssl | ||
${EXTERNAL_PROJECT_LOG_ARGS} | ||
GIT_REPOSITORY "https://github.com/openssl/openssl" | ||
GIT_TAG "c35608e5422d2718868d88439e22369d4aabb7c6" | ||
PREFIX ${OPENSSL_SOURCES_DIR} | ||
UPDATE_COMMAND "" | ||
INSTALL_DIR ${OPENSSL_INSTALL_DIR} | ||
CONFIGURE_COMMAND ./config -fPIC no-shared --prefix=${OPENSSL_INSTALL_DIR} | ||
BUILD_IN_SOURCE 1 | ||
BUILD_COMMAND $(MAKE) -j 12 | ||
INSTALL_COMMAND $(MAKE) install | ||
) | ||
|
||
ADD_DEPENDENCIES(extern_openssl zlib) | ||
ADD_LIBRARY(ssl STATIC IMPORTED GLOBAL) | ||
ADD_LIBRARY(crypto STATIC IMPORTED GLOBAL) | ||
SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_LIBRARIES}) | ||
SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${CRYPTO_LIBRARIES}) | ||
ADD_DEPENDENCIES(ssl extern_ssl) | ||
|
||
|
||
LIST(APPEND external_project_dependencies ssl) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -68,7 +68,7 @@ std::ostream &operator<<(std::ostream &os, const LoDTensor &t) { | |
// only print first ten elements | ||
int64_t size = t.numel() < 10 ? t.numel() : 10; | ||
for (int64_t i = 0; i < size; ++i) { | ||
if (t.type().hash_code() == typeid(float).hash_code()) { | ||
if (t.type().hash_code() == typeid(float).hash_code()) { // NOLINT | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can put these fix in another PR to reduce this PR's size? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ye.Will do it! |
||
os << t.data<float>()[i] << " "; | ||
} else if (t.type().hash_code() == typeid(int64_t).hash_code()) { | ||
os << t.data<int64_t>()[i] << " "; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,14 +14,22 @@ | |
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. macros.h文件是只给rpc相关使用,还是其他OP也能使用, 如果其他OP也能用,不需要RPC相关的OP引入RPC相关的头文件感觉不合适 |
||
#pragma once | ||
|
||
#ifdef PADDLE_WITH_DISTRIBUTE | ||
|
||
#ifdef PADDLE_WITH_GRPC | ||
|
||
#include "paddle/fluid/operators/distributed/grpc_client.h" | ||
#include "paddle/fluid/operators/distributed/grpc_server.h" | ||
#define RPCSERVER_T distributed::AsyncGRPCServer | ||
#define RPCCLIENT_T distributed::GRPCClient | ||
#else | ||
#define RPCSERVER_T paddle::operators::distributed::AsyncGRPCServer | ||
#define RPCCLIENT_T paddle::operators::distributed::GRPCClient | ||
|
||
#else // PADDLE_WITH_GRPC | ||
|
||
#include "paddle/fluid/operators/distributed/brpc_client.h" | ||
#include "paddle/fluid/operators/distributed/brpc_server.h" | ||
#define RPCSERVER_T distributed::AsyncBRPCServer | ||
#define RPCCLIENT_T distributed::BRPCClient | ||
#endif | ||
#define RPCSERVER_T paddle::operators::distributed::AsyncBRPCServer | ||
#define RPCCLIENT_T paddle::operators::distributed::BRPCClient | ||
|
||
#endif // PADDLE_WITH_GRPC | ||
|
||
#endif // PADDLE_WITH_DISTRIBUTE |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,33 +1,42 @@ | ||
if(NOT WITH_DISTRIBUTE) | ||
return() | ||
endif() | ||
|
||
if(WITH_GRPC) | ||
grpc_library(sendrecvop_grpc SRCS bytebuffer_stream.cc sendrecvop_utils.cc grpc_client.cc | ||
request_handler_impl.cc rpc_client.cc rpc_server.cc grpc_server.cc variable_response.cc PROTO send_recv.proto DEPS lod_tensor | ||
selected_rows memory) | ||
set(cc_generic_services "false") | ||
else() | ||
set(cc_generic_services "true") | ||
endif() | ||
configure_file(send_recv.proto.in ${CMAKE_CURRENT_SOURCE_DIR}/send_recv.proto @ONLY) | ||
|
||
if(WITH_GRPC) | ||
grpc_library(sendrecvop_grpc SRCS grpc_bytebuffer_stream.cc sendrecvop_utils.cc grpc_client.cc | ||
request_handler_impl.cc rpc_client.cc rpc_server.cc grpc_server.cc variable_response.cc grpc_variable_response.cc grpc_sendrecvop_utils.cc | ||
PROTO send_recv.proto | ||
DEPS lod_tensor selected_rows memory) | ||
set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor") | ||
set_source_files_properties(grpc_serde_test.cc rpc_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) | ||
cc_test(serde_test SRCS grpc_serde_test.cc variable_response.cc DEPS grpc++_unsecure grpc_unsecure gpr | ||
cares zlib protobuf sendrecvop_grpc SERIAL) | ||
cc_test(grpc_server_test SRCS rpc_server_test.cc DEPS sendrecvop_grpc | ||
grpc++_unsecure grpc_unsecure gpr cares zlib protobuf executor | ||
proto_desc lookup_table_op SERIAL) | ||
cc_test(grpc_serde_test SRCS grpc_serde_test.cc | ||
DEPS grpc++_unsecure grpc_unsecure gpr cares zlib protobuf sendrecvop_grpc SERIAL) | ||
cc_test(grpc_server_test SRCS rpc_server_test.cc | ||
DEPS sendrecvop_grpc grpc++_unsecure grpc_unsecure gpr cares zlib protobuf executor proto_desc lookup_table_op SERIAL) | ||
return() | ||
endif() | ||
|
||
|
||
set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor") | ||
set_source_files_properties(brpc_server.cc brpc_client.cc rpc_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) | ||
brpc_library(sendrecvop_brpc SRCS brpc_client.cc brpc_server.cc rpc_server.cc rpc_client.cc request_handler_impl.cc | ||
set_source_files_properties(brpc_server.cc brpc_client.cc rpc_server_test.cc brpc_serde_test.cc | ||
brpc_variable_response.cc brpc_sendrecvop_utils.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) | ||
|
||
brpc_library(sendrecvop_brpc SRCS brpc_client.cc brpc_server.cc rpc_server.cc rpc_client.cc request_handler_impl.cc brpc_sendrecvop_utils.cc | ||
brpc_variable_response.cc variable_response.cc sendrecvop_utils.cc | ||
PROTO send_recv.proto | ||
DEPS lod_tensor selected_rows memory) | ||
|
||
find_library(OPENSSL_CRYPTO_LIBRARY_STATIC NAMES libcrypto.so) | ||
ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL) | ||
SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY_STATIC}) | ||
|
||
set(brpc_test_depends sendrecvop_brpc brpc protobuf leveldb gflags glog executor proto_desc lookup_table_op snappystream snappy ssl crypto) | ||
|
||
find_library(OPENSSL_SSL_LIBRARY_STATIC NAMES libssl.so) | ||
ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL) | ||
SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY_STATIC}) | ||
cc_test(brpc_server_test SRCS rpc_server_test.cc | ||
DEPS ${brpc_test_depends} SERIAL) | ||
|
||
cc_test(brpc_server_test SRCS rpc_server_test.cc DEPS sendrecvop_brpc | ||
brpc protobuf leveldb gflags glog | ||
protobuf executor proto_desc lookup_table_op snappystream snappy ssl crypto SERIAL) | ||
cc_test(brpc_serde_test SRCS brpc_serde_test.cc | ||
DEPS ${brpc_test_depends} SERIAL) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no need to be different?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks! Done!