build(protobuf): allow to use system protobuf

Protobuf is not really needed by some backend, so allow to not build it. For example, the jetson build doesn't really need it.
jolibrain · Mar 8, 2021 · f38e209 · f38e209
1 parent 9de5f55
commit f38e209
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 21 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -21,6 +21,7 @@ option(USE_SIMSEARCH "build index and search services" OFF)
 option(USE_ANNOY "use annoy as indexer" OFF)
 option(USE_FAISS "use FAISS as indexer" ON)
 option(BUILD_SPDLOG "build SPDLOG instead of using system library" ON)
+option(BUILD_PROTOBUF "build PROTOBUF instead of using system library" ON)
 option(USE_BOOST_BACKTRACE "use boost backtrace" ON)
 
 if (USE_CAFFE)
@@ -122,34 +123,48 @@ endif()
 # PROTOBUF
 
 # use protobuf same version as pytorch one
-set(PROTOBUF_VERSION 3.11.4)
-ExternalProject_Add(
-   protobuf
-   PREFIX protobuf
-   URL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
-   URL_HASH SHA256=ba4bbc3e6b58d2ccfe406e616576ef48710a2aae205f463618025fc691549cfe
-   SOURCE_SUBDIR cmake
-   CMAKE_ARGS
-    -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
-    -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-    -DBUILD_SHARED_LIBS:BOOL=ON
-    -Dprotobuf_BUILD_TESTS:BOOL=OFF
-    -Dprotobuf_BUILD_EXAMPLES:BOOL=OFF
-   INSTALL_COMMAND ""
-)
-set(PROTOBUF_INCLUDE_DIR ${CMAKE_BINARY_DIR}/protobuf/src/protobuf/src)
-set(PROTOBUF_LIB_DIR ${CMAKE_BINARY_DIR}/protobuf/src/protobuf-build/)
-set(PROTOBUF_LIB_DEPS ${PROTOBUF_LIB_DIR}/libprotobuf.so ${PROTOBUF_LIB_DIR}/libprotoc.so ${PROTOBUF_LIB_DIR}/libprotobuf-lite.so)
-link_directories(${PROTOBUF_LIB_DIR})
-include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
+if (BUILD_PROTOBUF)
+    set(PROTOBUF_VERSION 3.11.4)
+    ExternalProject_Add(
+        protobuf
+        PREFIX protobuf
+        URL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
+        URL_HASH SHA256=ba4bbc3e6b58d2ccfe406e616576ef48710a2aae205f463618025fc691549cfe
+        SOURCE_SUBDIR cmake
+        CMAKE_ARGS
+        -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
+        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+        -DBUILD_SHARED_LIBS:BOOL=ON
+        -Dprotobuf_BUILD_TESTS:BOOL=OFF
+        -Dprotobuf_BUILD_EXAMPLES:BOOL=OFF
+        INSTALL_COMMAND ""
+    )
+    set(PROTOBUF_INCLUDE_DIR ${CMAKE_BINARY_DIR}/protobuf/src/protobuf/src)
+    set(PROTOBUF_LIB_DIR ${CMAKE_BINARY_DIR}/protobuf/src/protobuf-build/)
+    set(PROTOBUF_PROTOC ${PROTOBUF_LIB_DIR}/protoc)
+    set(PROTOBUF_LIB_DEPS ${PROTOBUF_LIB_DIR}/libprotobuf.so ${PROTOBUF_LIB_DIR}/libprotoc.so ${PROTOBUF_LIB_DIR}/libprotobuf-lite.so)
+    link_directories(${PROTOBUF_LIB_DIR})
+    include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
+else()
+    if (USE_CAFFE OR USE_TORCH)
+        find_package(Protobuf REQUIRED 3.11.4)
+    else()
+        find_package(Protobuf REQUIRED)
+    endif()
+
+    set(PROTOBUF_PROTOC ${Protobuf_PROTOC_EXECUTABLE})
+    set(PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIRS})
+    set(PROTOBUF_LIB_DEPS protobuf::libprotobuf)
+    add_custom_target(protobuf)
+endif()
 
 file(
     DOWNLOAD https://raw.githubusercontent.com/jolibrain/caffe/master/src/caffe/proto/caffe.proto ${CMAKE_BINARY_DIR}/caffe.proto
     EXPECTED_HASH SHA256=e82a1d51635be93e964c6debcb4cd76514fada46348fa7b30355c14850767360
 )
 add_custom_command(
     OUTPUT ${CMAKE_BINARY_DIR}/src/caffe.pb.h ${CMAKE_BINARY_DIR}/src/caffe.pb.cc
-    COMMAND LD_LIBRARY_PATH=${PROTOBUF_LIB_DIR} ${PROTOBUF_LIB_DIR}/protoc --proto_path=${CMAKE_BINARY_DIR} --cpp_out=${CMAKE_BINARY_DIR}/src ${CMAKE_BINARY_DIR}/caffe.proto
+    COMMAND LD_LIBRARY_PATH=${PROTOBUF_LIB_DIR} ${PROTOBUF_PROTOC} --proto_path=${CMAKE_BINARY_DIR} --cpp_out=${CMAKE_BINARY_DIR}/src ${CMAKE_BINARY_DIR}/caffe.proto
     MAIN_DEPENDENCY ${CMAKE_BINARY_DIR}/caffe.proto
     COMMENT Generating caffe.pb.h and caffe.pb.cc
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
@@ -1153,6 +1168,7 @@ message(STATUS "USE_JSON_API:          ${USE_JSON_API}")
 message(STATUS "USE_HTTP_SERVER:       ${USE_HTTP_SERVER}")
 message(STATUS "USE_HTTP_SERVER_OATPP: ${USE_HTTP_SERVER_OATPP}")
 message(STATUS "BUILD_SPDLOG:          ${BUILD_SPDLOG}")
+message(STATUS "BUILD_PROTOBUF:        ${BUILD_PROTOBUF}")
 message(STATUS "BUILD_TESTS:           ${BUILD_TESTS}")
 message(STATUS "BUILD_TOOLS:           ${BUILD_TOOLS}")
 message(STATUS "USE_CAFFE:             ${USE_CAFFE}")

diff --git a/ci/Jenkinsfile-jetson-nano.unittests b/ci/Jenkinsfile-jetson-nano.unittests
@@ -37,6 +37,7 @@ cd build
 cmake .. \
     -DBUILD_TESTS=ON  \
     -DBUILD_SPDLOG=ON \
+    -DBUILD_PROTOBUF=OFF \
     -DUSE_HTTP_SERVER_OATPP=ON \
     -DUSE_CAFFE=OFF  \
     -DUSE_TENSORRT=ON  \

diff --git a/ci/devel-jetson-nano.Dockerfile b/ci/devel-jetson-nano.Dockerfile
@@ -16,6 +16,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     g++ \
     gcc-7 g++-7 \
     zlib1g-dev \
+    protobuf-compiler \
+    libprotobuf-dev \
     libgoogle-glog-dev \
     libgflags-dev \
     libeigen3-dev \