Failed to link Clang built static library

[moravas]

Hi,

I have a CMake project, where I build aws-lambda-cpp as external project with Clang-14:

`
set(TESSER_THIRDPARTY_CMAKE_CACHE_ARGS
-DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD}
-DCMAKE_INSTALL_PREFIX:PATH=${PROJECT_SOURCE_DIR}/bin
)

ExternalProject_Add(
aws_lambda_cpp
GIT_REPOSITORY https://github.com/awslabs/aws-lambda-cpp.git
GIT_TAG v0.2.7
SOURCE_DIR "${PROJECT_SOURCE_DIR}/aws-lambda-cpp"
BUILD_ALWAYS 1
UPDATE_COMMAND ""
CMAKE_CACHE_ARGS
${TESSER_THIRDPARTY_CMAKE_CACHE_ARGS}
)
`

when I link it against to my final executable, I get the following linker error:
/usr/bin/ld: /home/<...>bin/lib/libaws-lambda-runtime.a: error adding symbols: file format not recognized clang: error: linker command failed with exit code 1 (use -v to see invocation)

The CMake target:
target_link_libraries(${PROJECT_NAME} ${AWSSDK_LINK_LIBRARIES} Boost::json Boost::log Boost::log_setup mongo::mongocxx_static mongo::bsoncxx_static AWS::aws-lambda-runtime pthread odbc)

Can anybody help me, what did I wrong?

Please let me know, if additional information needed.

Thank you

[moravas]

Additional note: we have many other third parties, that are also build with same settings, and if I remove only this project and the relevant code, everything links fine

[hbobenicio]

I'm facing the same problem here. If your check the libaws-lambda-runtime.a it doesn't actually contain ELF binaries... it has LLVM IR bitcode (at least in my case). So using the native linker ld doesn't work. We should probably be using llvm's linker (lld) instead...

[hbobenicio]

Yeah, using lld's instead of the native ld works (it knows how to link LLVM IR bitcode archives as static libs).
Just had to use -DCMAKE_CXX_FLAGS="-fuse-ld=lld" in my project.

If you don't want to use lld, then probably you either should turn ENABLE_LTO option off or switch to gnu gcc.

aws-lambda-cpp/CMakeLists.txt

Line 7 in 095ff84

option(ENABLE_LTO "Enables link-time optimization, requires compiler support." ON)

[hbobenicio]

Here is some investigation...

$ clang++ --version
clang version 10.0.0-4ubuntu1 
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin

$ /usr/bin/clang++ -DAWS_LAMBDA_LOG=0 -I/tmp/lambda/aws-lambda-cpp/include -O3 -DNDEBUG -fno-exceptions -fno-rtti -fvisibility=hidden -fvisibility-inlines-hidden -Wall -Wextra -Werror -Wconversion -Wno-sign-conversion -MD -MT CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o -MF CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o.d -o CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o -c /tmp/lambda/aws-lambda-cpp/src/logging.cpp

$ file CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o
CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o: ELF 64-bit LSB relocatable, x86-64, version 1 (SYSV), not stripped

$ /usr/bin/clang++ -DAWS_LAMBDA_LOG=0 -I/tmp/lambda/aws-lambda-cpp/include -O3 -DNDEBUG -flto=thin -fno-exceptions -fno-rtti -fvisibility=hidden -fvisibility-inlines-hidden -Wall -Wextra -Werror -Wconversion -Wno-sign-conversion -MD -MT CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o -MF CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o.d -o CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o -c /tmp/lambda/aws-lambda-cpp/src/logging.cpp

$ file CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o
CMakeFiles/aws-lambda-runtime.dir/src/logging.cpp.o: LLVM IR bitcode

So this issue is probably related to this -flto=thin flag. At least in my clang++ this causes the compiler to emit LLVM IR bitcodes (.bc) instead of ELF native binaries.

https://clang.llvm.org/docs/ThinLTO.html

"In ThinLTO mode, as with regular LTO, clang emits LLVM bitcode after the compile phase"

[hbobenicio]

I'd suggest that this may be a nice addition to the FAQ & Troubleshooting.

[hbobenicio]

"Note that ld.bfd from binutils version 2.21.51.0.2 and above also supports LTO via plugins. However, usage of the LLVM gold plugin with ld.bfd is not tested and therefore not officially supported or recommended."

https://llvm.org/docs/GoldPlugin.html

[hbobenicio]

@marcomagdy , this is how to reproduce the error:

FROM ubuntu:20.04

RUN apt-get update \
 && DEBIAN_FRONTEND=noninteractive apt-get install -y git cmake clang libcurl4-openssl-dev \
 && rm -rf /var/lib/apt/lists/*

RUN git clone --branch=master --depth=1 https://github.com/awslabs/aws-lambda-cpp.git /aws-lambda-cpp \
 && mkdir /aws-lambda-cpp/build \
 && cd /aws-lambda-cpp/build \
 && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/lambda-install \
 && make -j4 \
 && make install \
 && make clean

WORKDIR /my-lambda

# CMakeLists.txt AS-IS from README.md
RUN printf '\
cmake_minimum_required(VERSION 3.9)\n\
set(CMAKE_CXX_STANDARD 11)\n\
project(demo LANGUAGES CXX)\n\
find_package(aws-lambda-runtime)\n\
add_executable(${PROJECT_NAME} "main.cpp")\n\
target_link_libraries(${PROJECT_NAME} PRIVATE AWS::aws-lambda-runtime)\n\
target_compile_features(${PROJECT_NAME} PRIVATE "cxx_std_11")\n\
target_compile_options(${PROJECT_NAME} PRIVATE "-Wall" "-Wextra")\n\
aws_lambda_package_target(${PROJECT_NAME})\n\
' > /my-lambda/CMakeLists.txt

# main.cpp AS-IS from README.md
RUN printf \
'#include <aws/lambda-runtime/runtime.h>\n\
\n\
using namespace aws::lambda_runtime;\n\
\n\
static invocation_response my_handler(invocation_request const& req)\n\
{\n\
    if (req.payload.length() > 42) {\n\
        return invocation_response::failure("error message here"/*error_message*/,\n\
                                            "error type here" /*error_type*/);\n\
    }\n\
\n\
    return invocation_response::success("json payload here" /*payload*/,\n\
                                        "application/json" /*MIME type*/);\n\
}\n\
\n\
int main()\n\
{\n\
    run_handler(my_handler);\n\
    return 0;\n\
}\n\
' > /my-lambda/main.cpp

RUN mkdir build \
 && cd build \
 && cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/lambda-install \
 && make -j4 VERBOSE=1 \
 && make clean

and this:

docker build . -t foo

gives me exactly this same error:

/usr/bin/ld: /lambda-install/lib/libaws-lambda-runtime.a: error adding symbols: file format not recognized

Notice that It's just a clean ubuntu with clang installed which I think it might be a pretty common setup.

Clang's with LTO enabled outputs LLVM IR bitcode inside libaws-lambda-runtime.a (you can ar x the archive and file *.o if you wanna check that). and the default linker in this setup is GNU ld (aka ld.bfd) which doesn't support LLVM IR bitcode. The user has some options to solve this:

1. Change the linker (to lld or any other that supports LLVM IR bitcode); or
2. use the LLVM gold plugin with ld (which is not recommended). AFAIK it's just a linker flag you can configure; or
3. disable LTO; or
4. use gnu gcc (g++)

Please let me know if I can help somehow in a deeper analysis.

Thank you for your time on this.

[bmoffatt]

Indeed the root cause of this is a conflicted toolchain.

Observations from playing with that Dockerfile

1. apt-get install -y clang doesn't bring in lld transitively, but the LTO check passes without warning! Plumbing through -fuse-ld=lld results in another round of errors for a user before identifying the missing dependency.
2. CMake on Ubuntu is choosing clang++ as the default compiler when g++ is absent, but isn't smart enough to do the same for ld.lld in the LTO enabled scenario.

So, if I were an Ubuntu user who also preferred clang to gcc, my Dockerfile would also include:

RUN apt-get install -y lld
RUN update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.lld" 10

[hbobenicio]

Yes, if using LTO I agree the best option is indeed to use lld, either by setting the ld symbolic link like you did or just passing it to the compiler flags for this project build specifically with -DCMAKE_CXX_FLAGS="-fuse-ld=lld". As I almost always use ld for my compilations (and I've never really ran into problems until now caused by LTO) I've sticked with the second option. Disabling LTO is another option but I don't really know how this could affect performance and bundle sizes for this project...

Anyway, thanks for the analysis.

This is what motivated me in trying to add a faq & troubleshooting section for helping solving this.

[marcomagdy]

Note that if you build with Clang and GNU's ld with LTO turned on, you are able to produce a shared-library (.so file) for the runtime just fine.

[marcomagdy]

Moreover, if you enable LTO in the app, GNU's ld links it all just fine. I just tried it.
If you do not enable LTO in the app, then yes, you would get the error adding symbols: file format not recognized error.

So, it has nothing to do with GNU's ld not understanding LLVM IR as mentioned in the now merged README PR.

cc @bmoffatt

For posterity, here's how to enable LTO in CMake:

set_property(TARGET ${PROJECT_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)

[marcomagdy]

I think the right thing to do at this point is:

1. Document that LTO must to be turned on for the whole binary (if you're building the runtime as a static library).
2. Change LTO to be opt-in rather than opt-out (i.e. change the default setting in CMake).

I think this problem bites a lot of people because CMake currently (unless explicitly told otherwise) enables LTO and builds the runtime as a static library by default.

And since we don't have evidence that LTO makes a substantial difference, it's best to change that default rather than building the library as a shared object.

[hbobenicio]

Moreover, if you enable LTO in the app, GNU's ld links it all just fine. I just tried it.

Yes, this works because it will use the LLVM GoldPlugin with Gnu ld.
You can check that by adding this to cmake:

set(CMAKE_CXX_FLAGS "-v")

You would see this:

/usr/bin/c++  -v -g -flto=thin   CMakeFiles/demo.dir/main.cpp.o  -o demo  /lambda-install/lib/libaws-lambda-runtime.a /usr/lib/x86_64-linux-gnu/libcurl.so
[...]
"/usr/bin/ld" -z relro --hash-style=gnu --build-id --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o demo /usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu/crt1.o /usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu/crti.o /usr/bin/../lib/gcc/x86_64-linux-gnu/9/crtbegin.o -L/usr/bin/../lib/gcc/x86_64-linux-gnu/9 -L/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu -L/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/usr/lib/../lib64 -L/usr/lib/x86_64-linux-gnu/../../lib64 -L/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../.. -L/usr/lib/llvm-10/bin/../lib -L/lib -L/usr/lib -plugin /usr/lib/llvm-10/bin/../lib/LLVMgold.so -plugin-opt=mcpu=x86-64 -plugin-opt=thinlto CMakeFiles/demo.dir/main.cpp.o /lambda-install/lib/libaws-lambda-runtime.a /usr/lib/x86_64-linux-gnu/libcurl.so -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/bin/../lib/gcc/x86_64-linux-gnu/9/crtend.o /usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu/crtn.

Note the -plugin-opt=thinlto and -plugin LLVMgold.so flags there.

This is what I meant in the item 2:

"2. use the LLVM gold plugin with ld (which is not recommended). AFAIK it's just a linker flag you can configure; or"

I also quoted this from its own docs:

"Note that ld.bfd from binutils version 2.21.51.0.2 and above also supports LTO via plugins. However, usage of the LLVM gold plugin with ld.bfd is not tested and therefore not officially supported or recommended."

So it works but LLVM doesn't actually recommend this approach, unfortunately.

I totally agree with your final reasoning here:

"2. Change LTO to be opt-in rather than opt-out (i.e. change the default setting in CMake)."
"And since we don't have evidence that LTO makes a substantial difference, it's best to change that default rather than building the library as a shared object."

Thanks once again for this whole analysis session.

[marcomagdy]

Thank you for your patience and for following up.