Trouble Building Protobuf Project on Mac

[Anefu]

What version of protobuf and what language are you using?
Version: 25.3_1
Language: C++

What operating system (Linux, Windows, ...) and version?
Mac, running MacOS 14.4.1 Sonoma
What runtime / compiler are you using (e.g., python version or gcc version)
Apple clang version 15.0.0 (clang-1500.3.9.4)
Target: arm64-apple-darwin23.4.0
Thread model: posix
InstalledDir: /Library/Developer/CommandLineTools/usr/bin
What did you do?
Steps to reproduce the behavior:

1. Create a .proto file
2. Create a simple implementation to use the file
3. Compile using CMake
4. See error

What did you expect to see
Successful compilation
What did you see instead?

Undefined symbols for architecture arm64:
  "void absl::lts_20240116::log_internal::LogMessage::CopyToEncodedBuffer<(absl::lts_20240116::log_internal::LogMessage::StringType)0>(std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
      absl::lts_20240116::log_internal::LogMessage& absl::lts_20240116::log_internal::LogMessage::operator<<<19>(char const (&) [19]) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::LogMessage& absl::lts_20240116::log_internal::LogMessage::operator<<<unsigned long, 0>(unsigned long const&)", referenced from:
      absl::lts_20240116::log_internal::LogMessage::operator<<(unsigned long) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::LogMessageFatal::LogMessageFatal(char const*, int, std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
      GyronicsAPI::IMU6DOF::SharedDtor() in libproto.a[2](readapi.pb.cc.o)
      GyronicsAPI::IMU6DOF::MergeImpl(google::protobuf::Message&, google::protobuf::Message const&) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::LogMessageFatal::~LogMessageFatal()", referenced from:
      GyronicsAPI::IMU6DOF::SharedDtor() in libproto.a[2](readapi.pb.cc.o)
      GyronicsAPI::IMU6DOF::SharedDtor() in libproto.a[2](readapi.pb.cc.o)
      GyronicsAPI::IMU6DOF::MergeImpl(google::protobuf::Message&, google::protobuf::Message const&) in libproto.a[2](readapi.pb.cc.o)
      GyronicsAPI::IMU6DOF::MergeImpl(google::protobuf::Message&, google::protobuf::Message const&) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
      google::protobuf::internal::ArenaAlignAs(unsigned long) in libproto.a[2](readapi.pb.cc.o)
      ...
  "std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>* absl::lts_20240116::log_internal::MakeCheckOpString<void const*, void const*>(void const*, void const*, char const*)", referenced from:
      std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>* absl::lts_20240116::log_internal::Check_NEImpl<GyronicsAPI::IMU6DOF const*, GyronicsAPI::IMU6DOF*>(GyronicsAPI::IMU6DOF const* const&, GyronicsAPI::IMU6DOF* const&, char const*) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::CheckOpMessageBuilder::ForVar2()", referenced from:
      std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>* absl::lts_20240116::log_internal::MakeCheckOpString<unsigned long long, long long>(unsigned long long, long long, char const*) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::CheckOpMessageBuilder::NewString()", referenced from:
      std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>* absl::lts_20240116::log_internal::MakeCheckOpString<unsigned long long, long long>(unsigned long long, long long, char const*) in libproto.a[2](readapi.pb.cc.o)
  "absl::lts_20240116::log_internal::CheckOpMessageBuilder::CheckOpMessageBuilder(char const*)", referenced from:
      std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>* absl::lts_20240116::log_internal::MakeCheckOpString<unsigned long long, long long>(unsigned long long, long long, char const*) in libproto.a[2](readapi.pb.cc.o)
ld: symbol(s) not found for architecture arm64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
make[2]: *** [reader] Error 1
make[1]: *** [CMakeFiles/reader.dir/all] Error 2
make: *** [all] Error 2

Make sure you include information that can help us debug (full error message, exception listing, stack trace, logs).

Anything else we should know about your project / environment

[mkruskal-google]

What version of Abseil are you using? This looks like you might have an older one than 25.x requires

[wizetLee]

#12292 (comment)
It helped me solve the same problem

[Anefu]

What version of Abseil are you using? This looks like you might have an older one than 25.x requires

Abseil version: 20240116.1

[Anefu]

#12292 (comment) It helped me solve the same problem

Thanks @wizetLee but this still doesn't help.

[mkruskal-google]

How are you including protobuf in cmake? I believe the problem back then was how find_package was being used. i.e. this:

find_package(protobuf REQUIRED)

does not work because it uses the deprecated FindProtobuf file we don't own. Instead, you need

find_package(protobuf REQUIRED CONFIG)

[Anefu]

Here's a portion of my CMakeList.txt:

option(protobuf_MODULE_COMPATIBLE TRUE)
find_package(Protobuf CONFIG REQUIRED)
find_program(_PROTOBUF_PROTOC protoc)
find_package(Boost 1.83 COMPONENTS system thread REQUIRED)
    find_package(absl CONFIG REQUIRED)
    set(CMAKE_CXX_STANDARD 20)
    set(CMAKE_CXX_STANDARD_REQUIRED ON)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20 -fsanitize=address")

    get_filename_component(api_proto "websockets/api.proto" ABSOLUTE)
    get_filename_component(api_proto_path "${api_proto}" PATH)

    add_custom_command(
      OUTPUT "${api_proto_srcs}" "${api_proto_hdrs}"
      COMMAND ${_PROTOBUF_PROTOC}
      ARGS
        --cpp_out "${PROJECT_BINARY_DIR}"
        -I "${api_proto_path}"
        "${api_proto}"
      DEPENDS "${api_proto}")

    add_library(proto
        ${api_proto_srcs}
        ${api_proto_hdrs}
    )

    target_link_libraries(proto
        PUBLIC protobuf::libprotobuf
    )

[fruffy]

How are you including protobuf in cmake? I believe the problem back then was how find_package was being used. i.e. this:

find_package(protobuf REQUIRED)

does not work because it uses the deprecated FindProtobuf file we don't own. Instead, you need

find_package(protobuf REQUIRED CONFIG)

Thanks, I think it would be useful to make this more prominent in the docs and also make clear that the variables defined in https://cmake.org/cmake/help/latest/module/FindProtobuf.html are not available. On that note, what are the canonical Protobuf CMake variables we can actually use? We are still struggling with this.

Overall, this is causing significant confusion and breakage with many open-source repositories trying to use Protobuf.

[chazeon]

Not sure if this is the right place to ask, but I am having exactly the same issue while building a program against Tensorflow, which uses Protobuf.

cd /Users/chazeon/Documents/Projects/deepmd-kit/source/op && /Users/chazeon/miniforge3/envs/deepmd-dev/bin/cmake -E cmake_link_script CMakeFiles/deepmd_op.dir/link.txt --verbose=1
/Users/chazeon/miniforge3/envs/deepmd-dev/bin/arm64-apple-darwin20.0.0-clang++ -ftree-vectorize -fPIC -fstack-protector-strong -O2 -pipe -stdlib=libc++ -fvisibility-inlines-hidden -fmessage-length=0 -isystem /Users/chazeon/miniforge3/envs/deepmd-dev/include -fopenmp=libomp -I -O3 -DNDEBUG -arch arm64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -bundle -Wl,-headerpad_max_install_names -Wl,-headerpad_max_install_names -Wl,-dead_strip_dylibs -Wl,-rpath,/Users/chazeon/miniforge3/envs/deepmd-dev/lib -L/Users/chazeon/miniforge3/envs/deepmd-dev/lib -o libdeepmd_op.so CMakeFiles/deepmd_op.dir/add_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/copy_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/custom_op.cc.o CMakeFiles/deepmd_op.dir/descrpt.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef_para.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef_vert.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_mask.cc.o CMakeFiles/deepmd_op.dir/dotmul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/ewald_recp.cc.o CMakeFiles/deepmd_op.dir/flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/gelu_multi_device.cc.o CMakeFiles/deepmd_op.dir/map_aparam.cc.o CMakeFiles/deepmd_op.dir/map_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_fitnet_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_flt2fix_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/mul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/neighbor_stat.cc.o CMakeFiles/deepmd_op.dir/pair_tab.cc.o CMakeFiles/deepmd_op.dir/pairwise.cc.o CMakeFiles/deepmd_op.dir/prod_env_mat_multi_device.cc.o CMakeFiles/deepmd_op.dir/prod_env_mat_multi_device_nvnmd.cc.o CMakeFiles/deepmd_op.dir/prod_force.cc.o CMakeFiles/deepmd_op.dir/prod_force_multi_device.cc.o CMakeFiles/deepmd_op.dir/prod_force_se_a_mask.cc.o CMakeFiles/deepmd_op.dir/prod_virial.cc.o CMakeFiles/deepmd_op.dir/prod_virial_multi_device.cc.o CMakeFiles/deepmd_op.dir/quantize_nvnmd.cc.o CMakeFiles/deepmd_op.dir/soft_min.cc.o CMakeFiles/deepmd_op.dir/soft_min_force.cc.o CMakeFiles/deepmd_op.dir/soft_min_virial.cc.o CMakeFiles/deepmd_op.dir/tabulate_multi_device.cc.o CMakeFiles/deepmd_op.dir/tanh4_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/unaggregated_grad.cc.o CMakeFiles/deepmd_op.dir/optimizer/parallel.cc.o  -Wl,-rpath,/Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow -Wl,-rpath,/Users/chazeon/Documents/Projects/deepmd-kit/source/lib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/libomp.dylib -lProtobuf /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/libtensorflow_framework.2.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/libtensorflow_cc.2.dylib ../lib/libdeepmd.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/../../..//libprotobuf.24.4.0.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/libomp.dylib 
Undefined symbols for architecture arm64:
  "void absl::lts_20230802::log_internal::LogMessage::CopyToEncodedBuffer<(absl::lts_20230802::log_internal::LogMessage::StringType)0>(std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
  "absl::lts_20230802::log_internal::LogMessage::operator<<(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&)", referenced from:
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
  "absl::lts_20230802::log_internal::LogMessageFatal::LogMessageFatal(char const*, int, std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
  "absl::lts_20230802::log_internal::LogMessageFatal::~LogMessageFatal()", referenced from:
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
      tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
  "absl::lts_20230802::hash_internal::MixingHashState::LowLevelHashImpl(unsigned char const*, unsigned long)", referenced from:
      absl::lts_20230802::hash_internal::MixingHashState::combine_contiguous(absl::lts_20230802::hash_internal::MixingHashState, unsigned char const*, unsigned long) in parallel.cc.o
  "absl::lts_20230802::hash_internal::MixingHashState::CombineLargeContiguousImpl64(unsigned long long, unsigned char const*, unsigned long)", referenced from:
      absl::lts_20230802::hash_internal::MixingHashState::combine_contiguous(absl::lts_20230802::hash_internal::MixingHashState, unsigned char const*, unsigned long) in parallel.cc.o
  "absl::lts_20230802::hash_internal::MixingHashState::kSeed", referenced from:
      google::protobuf::internal::KeyMapBase<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>::FindHelper(std::__1::basic_string_view<char, std::__1::char_traits<char>>, absl::lts_20230802::container_internal::btree_iterator<absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<google::protobuf::internal::VariantKey, google::protobuf::internal::NodeBase*, std::__1::less<google::protobuf::internal::VariantKey>, google::protobuf::internal::MapAllocator<std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>>, 256, false>>, std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>&, std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>*>*) const in parallel.cc.o
      google::protobuf::internal::KeyMapBase<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>::TransferList(google::protobuf::internal::KeyNode<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>*) in parallel.cc.o
  "absl::lts_20230802::Status::UnrefNonInlined(unsigned long)", referenced from:
      absl::lts_20230802::Status::~Status() in add_flt_nvnmd.cc.o
      absl::lts_20230802::Status::~Status() in copy_flt_nvnmd.cc.o
      absl::lts_20230802::Status::~Status() in custom_op.cc.o
      DescrptOp<Eigen::ThreadPoolDevice, float>::DescrptOp(tensorflow::OpKernelConstruction*) in descrpt.cc.o
      absl::lts_20230802::Status::~Status() in descrpt.cc.o
      DescrptOp<Eigen::ThreadPoolDevice, double>::DescrptOp(tensorflow::OpKernelConstruction*) in descrpt.cc.o
      DescrptSeAEfOp<Eigen::ThreadPoolDevice, float>::DescrptSeAEfOp(tensorflow::OpKernelConstruction*) in descrpt_se_a_ef.cc.o
      ...
  "absl::lts_20230802::Status::Status(absl::lts_20230802::StatusCode, std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
      absl::lts_20230802::Status tsl::errors::Internal<char const*, char const*, char const*, char const*, char const*, int>(char const*, char const*, char const*, char const*, char const*, int) in custom_op.cc.o
      absl::lts_20230802::Status tsl::errors::ResourceExhausted<char const*, char const*, char const*, char const*, char const*, int>(char const*, char const*, char const*, char const*, char const*, int) in custom_op.cc.o
      absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt.cc.o
      absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef.cc.o
      absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef_para.cc.o
      absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef_vert.cc.o
      absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_mask.cc.o
      ...
ld: symbol(s) not found for architecture arm64
clang-16: error: linker command failed with exit code 1 (use -v to see invocation)
make[2]: *** [op/CMakeFiles/deepmd_op.dir/build.make:751: op/libdeepmd_op.so] Error 1
make[2]: Leaving directory '/Users/chazeon/Documents/Projects/deepmd-kit/source'
make[1]: *** [CMakeFiles/Makefile2:277: op/CMakeFiles/deepmd_op.dir/all] Error 2
make[1]: Leaving directory '/Users/chazeon/Documents/Projects/deepmd-kit/source'
make: *** [Makefile:136: all] Error 2

my Tensorflow and Protobuf is installed via mamba

(deepmd-dev) (.env) chazeon@Chenxings-MBP [23:06:17] [~/Documents/Projects/deepmd-kit/source] [devel *]
-> % mamba list | grep tensorflow
libtensorflow             2.15.0           cpu_h9d8c315_2    conda-forge
libtensorflow_cc          2.15.0           cpu_h1998a8a_2    conda-forge
tensorflow                2.15.0          cpu_py311h9b82b1a_2    conda-forge
tensorflow-base           2.15.0          cpu_py311he034567_2    conda-forge
tensorflow-estimator      2.15.0          cpu_py311hcceb7c1_2    conda-forge
(deepmd-dev) (.env) chazeon@Chenxings-MBP [23:06:46] [~/Documents/Projects/deepmd-kit/source] [devel *]
-> % mamba list | grep protobuf  
libprotobuf               4.24.4               h810fc01_0    conda-forge
protobuf                  4.24.4          py311h4d1eceb_0    conda-forge

I have to link the following libraries explicitly to make the build work:

find_package(absl REQUIRED)
target_link_libraries(
    ${libname} PRIVATE
    absl::log_internal_message
    absl::log_internal_check_op
    absl::status
)

I notice that the symbol

void absl::lts_20240116::log_internal::LogMessage::CopyToEncodedBuffer<(absl::lts_20240116::log_internal::LogMessage::StringType)0>(std::__1::basic_string_view<char, std::__1::char_traits<char>>)

is not publicly exposed from libprotobuf.dylib.