From c81fdf99d09eaecf1165e19e75f796e7173fec73 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 10 Nov 2025 12:20:13 +0800 Subject: [PATCH 1/3] HADOOP-19605. (3.4) Upgrade Protobuf 3.25.5 for docker images (#7780) * HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images. Reviewed-by: Steve Loughran Signed-off-by: Shilun Fan --- BUILDING.txt | 57 +++++++++------- dev-support/docker/Dockerfile | 2 +- dev-support/docker/Dockerfile_aarch64 | 2 +- dev-support/docker/Dockerfile_centos_8 | 2 +- dev-support/docker/Dockerfile_debian_10 | 2 +- .../docker/pkg-resolver/install-protobuf.sh | 19 +++--- .../src/main/native/libhdfspp/CMakeLists.txt | 68 ++++++++++++++++++- .../native/libhdfspp/lib/proto/CMakeLists.txt | 5 +- .../libhdfspp/lib/proto/protoc_gen_hrpc.cc | 1 - .../native/libhdfspp/tests/CMakeLists.txt | 30 ++++---- .../tests/remote_block_reader_test.cc | 5 +- .../native/libhdfspp/tests/rpc_engine_test.cc | 1 - 12 files changed, 137 insertions(+), 57 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 1e2a1fef1a098..14cbb30f54c12 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ Requirements: * JDK 1.8 * Maven 3.3 or later * Boost 1.72 (if compiling native code) -* Protocol Buffers 3.21.12 (if compiling native code) +* Protocol Buffers 3.25.5 (if compiling native code) * CMake 3.19 or newer (if compiling native code) * Zlib devel (if compiling native code) * Cyrus SASL devel (if compiling native code) @@ -74,13 +74,15 @@ Refer to dev-support/docker/Dockerfile): $ ./bootstrap $ make -j$(nproc) $ sudo make install -* Protocol Buffers 3.21.12 (required to build native code) - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure - $ make -j$(nproc) - $ sudo make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ sudo cmake --install build * Boost $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 $ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0 @@ -430,14 +432,15 @@ Installing required dependencies for clean install of macOS 10.14: * Install native libraries, only openssl is required to compile native code, you may optionally install zlib, lz4, etc. $ brew install openssl -* Protocol Buffers 3.21.12 (required to compile native code) - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure - $ make - $ make check - $ make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ cmake --install build $ protoc --version Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken @@ -469,13 +472,15 @@ Building on CentOS 8 * Install python2 for building documentation. $ sudo dnf install python2 -* Install Protocol Buffers v3.21.12. - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure --prefix=/usr/local - $ make - $ sudo make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ sudo cmake --install build --prefix /usr/local $ cd .. * Install libraries provided by CentOS 8. @@ -525,9 +530,15 @@ Requirements: * Windows 10 * JDK 1.8 +<<<<<<< HEAD * Maven 3.0 or later (maven.apache.org) * Boost 1.72 (boost.org) * Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags) +======= +* Maven 3.3 or later (maven.apache.org) +* Boost 1.86.0 (boost.org) +* Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags) +>>>>>>> d361d0ddff2 (HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images (#7780)) * CMake 3.19 or newer (cmake.org) * Visual Studio 2019 (visualstudio.com) * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 3b71e622a575e..6917827335c15 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 26ad10edd6f23..8b999b191ae9a 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -67,7 +67,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index ee0c8e88f74e4..f4e9b380886b6 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -101,7 +101,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 index 71446b27f686b..1f6b296ee49f8 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh index f8319f6acea60..a512d5ee026dd 100644 --- a/dev-support/docker/pkg-resolver/install-protobuf.sh +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -27,29 +27,32 @@ if [ $? -eq 1 ]; then exit 1 fi -default_version="3.21.12" +default_version="3.25.5" version_to_install=$default_version if [ -n "$2" ]; then version_to_install="$2" fi -if [ "$version_to_install" != "3.21.12" ]; then +if [ "$version_to_install" != "3.25.5" ]; then echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" version_to_install=$default_version fi -if [ "$version_to_install" == "3.21.12" ]; then +if [ "$version_to_install" == "3.25.5" ]; then # hadolint ignore=DL3003 mkdir -p /opt/protobuf-src && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \ + https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz \ -o /opt/protobuf.tar.gz && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + curl -L -s -S \ + https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz \ + -o /opt/abseil-cpp.tar.gz && + tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C /opt/protobuf-src/third_party/abseil-cpp && cd /opt/protobuf-src && - ./autogen.sh && - ./configure --prefix=/opt/protobuf && - make "-j$(nproc)" && - make install && + cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF && + cmake --build build --parallel $(nproc) && + cmake --install build --prefix /opt/protobuf && cd /root && rm -rf /opt/protobuf-src else diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index a42ee2e211494..9562f668d876e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -43,9 +43,70 @@ SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{ # Specify PROTOBUF_HOME so that find_package picks up the correct version SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}") +if (BUILD_SHARED_LIBS AND MSVC) + # On MSVC Abseil is bundled into a single DLL. + # This condition is necessary as of abseil 20230125.3 when abseil is consumed via add_subdirectory, + # the abseil_dll target is named abseil_dll, while if abseil is consumed via find_package, the target + # is called absl::abseil_dll + # Once https://github.com/abseil/abseil-cpp/pull/1466 is merged and released in the minimum version of + # abseil required by protobuf, it is possible to always link absl::abseil_dll and absl::abseil_test_dll + # and remove the if + if(protobuf_ABSL_PROVIDER STREQUAL "package") + set(protobuf_ABSL_USED_TARGETS absl::abseil_dll) + set(protobuf_ABSL_USED_TEST_TARGETS absl::abseil_test_dll) + else() + set(protobuf_ABSL_USED_TARGETS abseil_dll) + set(protobuf_ABSL_USED_TEST_TARGETS abseil_test_dll) + endif() +else() + set(protobuf_ABSL_USED_TARGETS + absl::absl_check + absl::absl_log + absl::algorithm + absl::base + absl::bind_front + absl::bits + absl::btree + absl::cleanup + absl::cord + absl::core_headers + absl::debugging + absl::die_if_null + absl::dynamic_annotations + absl::flags + absl::flat_hash_map + absl::flat_hash_set + absl::function_ref + absl::hash + absl::layout + absl::log_initialize + absl::log_severity + absl::memory + absl::node_hash_map + absl::node_hash_set + absl::optional + absl::span + absl::status + absl::statusor + absl::strings + absl::synchronization + absl::time + absl::type_traits + absl::utility + absl::variant + utf8_range::utf8_validity + utf8_range::utf8_range + ) + set(protobuf_ABSL_USED_TEST_TARGETS + absl::scoped_mock_log + ) +endif () + find_package(Doxygen) find_package(OpenSSL REQUIRED) find_package(Protobuf REQUIRED) +find_package(absl REQUIRED) +find_package(utf8_range REQUIRED) find_package(CyrusSASL) find_package(GSasl) find_package(Threads) @@ -89,7 +150,10 @@ endif (NOT THREAD_LOCAL_SUPPORTED) # to compile some dummy code unset (PROTOC_IS_COMPATIBLE CACHE) set (CMAKE_REQUIRED_INCLUDES ${PROTOBUF_INCLUDE_DIRS}) -set (CMAKE_REQUIRED_LIBRARIES ${PROTOBUF_LIBRARY} ${PROTOBUF_PROTOC_LIBRARY}) +set (CMAKE_REQUIRED_LIBRARIES + ${PROTOBUF_LIBRARY} + ${PROTOBUF_PROTOC_LIBRARY} + ${protobuf_ABSL_USED_TARGETS}) check_cxx_source_compiles( "#include #include @@ -280,6 +344,7 @@ if (HADOOP_BUILD AND NOT MSVC) hadoop_target_link_dual_libraries(hdfspp ${LIB_DL} ${PROTOBUF_LIBRARY} + ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} @@ -291,6 +356,7 @@ else (HADOOP_BUILD AND NOT MSVC) target_link_libraries(hdfspp_static PUBLIC ${LIB_DL} ${PROTOBUF_LIBRARY} + ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt index 35a38258f5f49..545562b09e54f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt @@ -38,7 +38,10 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ) add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc) -target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY}) +target_link_libraries(protoc-gen-hrpc + ${PROTOBUF_PROTOC_LIBRARY} + ${PROTOBUF_LIBRARY} + ${protobuf_ABSL_USED_TARGETS}) function(GEN_HRPC SRCS) if(NOT ARGN) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc index e7355c0b8c191..28a9e4af80da2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc @@ -48,7 +48,6 @@ class StubGenerator : public CodeGenerator { bool StubGenerator::Generate(const FileDescriptor *file, const std::string &, GeneratorContext *ctx, std::string *) const { - namespace pb = ::google::protobuf; std::unique_ptr os( ctx->Open(StripProto(file->name()) + ".hrpc.inl")); Printer out(os.get(), '$'); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 3e52c6d965a01..101735fde3172 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -79,11 +79,11 @@ target_link_libraries(get_jni_test gmock_main hdfs_static ${CMAKE_THREAD_LIBS_IN add_memcheck_test(get_jni get_jni_test) add_executable(remote_block_reader_test remote_block_reader_test.cc) -target_link_libraries(remote_block_reader_test test_common reader proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(remote_block_reader_test test_common reader proto common connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(remote_block_reader remote_block_reader_test) add_executable(sasl_digest_md5_test sasl_digest_md5_test.cc) -target_link_libraries(sasl_digest_md5_test common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(sasl_digest_md5_test common ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(sasl_digest_md5 sasl_digest_md5_test) add_executable(retry_policy_test retry_policy_test.cc) @@ -92,15 +92,15 @@ add_memcheck_test(retry_policy retry_policy_test) include_directories(${CMAKE_CURRENT_BINARY_DIR}) add_executable(rpc_engine_test rpc_engine_test.cc ${PROTO_TEST_SRCS} ${PROTO_TEST_HDRS}) -target_link_libraries(rpc_engine_test test_common rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES}) +target_link_libraries(rpc_engine_test test_common rpc proto common ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES}) add_memcheck_test(rpc_engine rpc_engine_test) add_executable(bad_datanode_test bad_datanode_test.cc) -target_link_libraries(bad_datanode_test rpc reader proto fs bindings_c rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(bad_datanode_test rpc reader proto fs bindings_c rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(bad_datanode bad_datanode_test) add_executable(node_exclusion_test node_exclusion_test.cc) -target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(node_exclusion node_exclusion_test) add_executable(configuration_test $ configuration_test.cc) @@ -114,30 +114,30 @@ target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_L add_memcheck_test(hdfs_configuration hdfs_configuration_test) add_executable(hdfspp_errors_test hdfspp_errors.cc) -target_link_libraries(hdfspp_errors_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(hdfspp_errors_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(hdfspp_errors hdfspp_errors_test) add_executable(hdfs_builder_test $ hdfs_builder_test.cc) target_include_directories(hdfs_builder_test PRIVATE ../lib) -target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(hdfs_builder_test hdfs_builder_test) add_executable(logging_test logging_test.cc $) target_include_directories(logging_test PRIVATE ../lib) -target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(logging_test logging_test) add_executable(hdfs_ioservice_test hdfs_ioservice_test.cc) -target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(hdfs_ioservice hdfs_ioservice_test) add_executable(user_lock_test user_lock_test.cc) -target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(user_lock user_lock_test) add_executable(hdfs_config_connect_bugs_test $ hdfs_config_connect_bugs.cc) target_include_directories(hdfs_config_connect_bugs_test PRIVATE ../lib) -target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test) @@ -167,22 +167,22 @@ add_dependencies(hdfspp_test_static proto) #TODO: Link against full library rather than just parts build_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static expect.c test_libhdfs_threaded.c ${OS_DIR}/thread.c) -link_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES} ) +link_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES} ) add_libhdfs_test (libhdfs_threaded hdfspp_test_shim_static) build_libhdfs_test(hdfspp_mini_dfs_smoke hdfspp_test_shim_static ${CMAKE_CURRENT_LIST_DIR}/hdfspp_mini_dfs_smoke.cc) -link_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static fs reader rpc proto common connection gmock_main ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) +link_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static fs reader rpc proto common connection gmock_main ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) add_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static) if (NOT MSVC) build_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c) - link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) + link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) add_memcheck_test(libhdfs_mini_stress_valgrind_hdfspp_test_static libhdfs_mini_stress_valgrind_hdfspp_test_static) set_target_properties(libhdfs_mini_stress_valgrind_hdfspp_test_static PROPERTIES COMPILE_DEFINITIONS "VALGRIND") endif (NOT MSVC) build_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c) -link_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) +link_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${protobuf_ABSL_USED_TARGETS} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) add_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static) build_libhdfs_test(hdfs_ext hdfspp_test_shim_static ${CMAKE_CURRENT_LIST_DIR}/hdfs_ext_test.cc) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc index 0c22e89f4c388..c0b1244fc29f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc @@ -58,8 +58,7 @@ using ::testing::Return; using std::make_pair; using std::string; -namespace pb = ::google::protobuf; -namespace pbio = pb::io; +namespace pbio = ::google::protobuf::io; namespace hdfs { @@ -114,7 +113,7 @@ class PartialMockReader : public BlockReaderImpl { } -static inline string ToDelimitedString(const pb::MessageLite *msg) { +static inline string ToDelimitedString(const google::protobuf::MessageLite *msg) { string res; res.reserve(hdfs::DelimitedPBMessageSize(msg)); pbio::StringOutputStream os(&res); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc index ba09e8fc90781..1c9bc234e4b64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc @@ -48,7 +48,6 @@ using ::testing::Return; using ::std::make_pair; using ::std::string; -namespace pb = ::google::protobuf; namespace pbio = ::google::protobuf::io; namespace hdfs { From 89df71f36080b2c71545c6997864f8edb5325717 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 13 Nov 2025 03:38:17 +0800 Subject: [PATCH 2/3] resolve conflicts --- BUILDING.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 14cbb30f54c12..c27433ba9fdbb 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -530,15 +530,9 @@ Requirements: * Windows 10 * JDK 1.8 -<<<<<<< HEAD -* Maven 3.0 or later (maven.apache.org) -* Boost 1.72 (boost.org) -* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags) -======= * Maven 3.3 or later (maven.apache.org) -* Boost 1.86.0 (boost.org) +* Boost 1.72 (boost.org) * Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags) ->>>>>>> d361d0ddff2 (HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images (#7780)) * CMake 3.19 or newer (cmake.org) * Visual Studio 2019 (visualstudio.com) * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from From 2e898a98c0c7b5650a32bd1a7dd06d230918bdc9 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 13 Nov 2025 03:39:54 +0800 Subject: [PATCH 3/3] revert --- BUILDING.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILDING.txt b/BUILDING.txt index c27433ba9fdbb..5a5399e70d5ee 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -530,7 +530,7 @@ Requirements: * Windows 10 * JDK 1.8 -* Maven 3.3 or later (maven.apache.org) +* Maven 3.0 or later (maven.apache.org) * Boost 1.72 (boost.org) * Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags) * CMake 3.19 or newer (cmake.org)