llvm-tblgen and clang-tblgen: error while loading shared libraries: libc++.so.1

[arcivanov]

I have a custom two-stage build.

Stage 1 build LLVM 13.0.1 toolchain via GCC on Fedora 35 x86.
Stage 2 build LLVM 13.0.1 toolchain using Stage 1 relying completely on LLVM (libc++, compiler-rt, libunwind everywhere).

Stage 1 builds and functions fine.
Stage 2 fails to build while invoking any target with llvm-tblgen in its path, because it's built before libc++ it requires.
If llvm-tblgen is specified to be remote via "-DLLVM_TABLEGEN" then the identical thing happens to clang-tblgen.

IMPORTANT: The CMAKE_INSTALL_PREFIX of Stage 1 is on PATH when building Stage 2 but is in a user home directory

Stage 1 CMAKE cache:

set(LLVM_ENABLE_PROJECTS "clang;lld;compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")

set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "")

set(LLVM_BUILD_TOOLS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")

set(LLVM_INSTALL_BINUTILS_SYMLINKS ON CACHE BOOL "")
set(LLVM_INSTALL_CCTOOLS_SYMLINKS ON CACHE BOOL "")

set(LLVM_PARALLEL_LINK_JOBS 1 CACHE STRING "")

set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
set(CLANG_LINK_CLANG_DYLIB ON CACHE BOOL "")

set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")

set(CMAKE_BUILD_TYPE Release CACHE STRING "")
set(CMAKE_C_FLAGS_RELEASE "-O2" CACHE STRING "")
set(CMAKE_CXX_FLAGS_RELEASE "-O2" CACHE STRING "")
set(CMAKE_ASM_FLAGS_RELEASE "-O2" CACHE STRING "")

Stage 2 CMAKE cache:

set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;lldb" CACHE STRING "")
set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "")

set(LLVM_BUILD_TOOLS ON CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")

set(LLVM_ENABLE_DUMP ON CACHE BOOL "")

set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
set(CLANG_LINK_CLANG_DYLIB ON CACHE BOOL "")

set(LLVM_ENABLE_LTO Thin CACHE BOOL "")
set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
set(LLVM_ENABLE_LLD ON CACHE BOOL "")
set(LLVM_ENABLE_LIBCXX ON CACHE BOOL "")

set(CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
set(CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
set(CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")

set(COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
set(SANITIZER_CXX_ABI libc++ CACHE STRING "")
set(SANITIZER_TEST_CXX libc++ CACHE STRING "")
set(LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
set(LIBCXX_USE_COMPILER_RT YES CACHE BOOL "")
set(LIBCXXABI_USE_COMPILER_RT YES CACHE BOOL "")
set(LIBCXXABI_USE_LLVM_UNWINDER YES CACHE BOOL "")

set(LLVM_INSTALL_BINUTILS_SYMLINKS ON CACHE BOOL "")
set(LLVM_INSTALL_CCTOOLS_SYMLINKS ON CACHE BOOL "")

set(LLVM_PARALLEL_LINK_JOBS 1 CACHE STRING "")

set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")

set(CMAKE_C_COMPILER clang CACHE STRING "")
set(CMAKE_CXX_COMPILER clang++ CACHE STRING "")
set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(CMAKE_ASM_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")

Stage 1 and Stage 2 are built virtually identically.
Stage 2 is built as follows, where LLVM_DISTRO_CONF is the CMAKE cache file above:

cmake3 -G Ninja \
  -Wno-dev \
  -DCMAKE_INSTALL_PREFIX="$TARGET_DIR" \
  -DPYTHON_HOME="$(python -c 'import sys; print(sys.exec_prefix)')" \
  -DPYTHON_EXECUTABLE="$(python -c 'import sys; print(sys.executable)')" \
  -DPython3_EXECUTABLE="$(python -c 'import sys; print(sys.executable)')" \
  -C "$LLVM_DISTRO_CONF" \
  -S "$SOURCE_DIR" -B "$BUILD_DIR"

pushd "$BUILD_DIR"

ninja-build install-runtimes

The build fails with the above llvm-tblgen or clang-tblgen error.

Example:

[423/2897] Building Attributes.inc...
FAILED: include/llvm/IR/Attributes.inc /home/user/Documents/src/llvm/llvm.stage2.build/include/llvm/IR/Attributes.inc 
cd /home/user/Documents/src/llvm/llvm.stage2.build && /home/user/Documents/src/llvm/llvm.stage2.build/bin/llvm-tblgen -gen-attrs -I /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR -I/home/user/Documents/src/llvm/llvm.stage2.build/include -I/home/user/Documents/src/llvm/llvm-project/llvm/include /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR/Attributes.td --write-if-changed -o include/llvm/IR/Attributes.inc -d include/llvm/IR/Attributes.inc.d
/home/user/Documents/src/llvm/llvm.stage2.build/bin/llvm-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory

[arcivanov]

I guess the workaround would be to force llvm-tblgen and clang-tblgen to link statically, but I can't seem to find a way to do that.

[arcivanov]

The problem is reproducible for the stock LLVM 2-stage bootstrap build as well, so stage1 install location is NOT relevant.

Here's the example:

twostage.cmake

# Common
set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;lldb" CACHE STRING "")
set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")

set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
set(CLANG_BOOTSTRAP_PASSTHROUGH "CMAKE_INSTALL_PREFIX;PYTHON_HOME;PYTHON_EXECUTABLE;Python3_EXECUTABLE;LLVM_TARGETS_TO_BUILD;LLVM_PARALLEL_COMPILE_JOBS;LLVM_PARALLEL_LINK_JOBS" CACHE STRING "")

set(LLVM_PARALLEL_LINK_JOBS 1 CACHE STRING "")

# Stage 1
set(LLVM_BUILD_TOOLS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")

set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")

# Stage 2
set(BOOTSTRAP_LLVM_BUILD_TOOLS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")

set(BOOTSTRAP_LLVM_ENABLE_DUMP ON CACHE BOOL "")

set(BOOTSTRAP_LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_CLANG_LINK_CLANG_DYLIB ON CACHE BOOL "")

set(BOOTSTRAP_LLVM_ENABLE_LTO THIN CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBC ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBCXX ON CACHE BOOL "")

set(BOOTSTRAP_CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_LINKER lld CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")

set(BOOTSTRAP_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
set(BOOTSTRAP_SANITIZER_CXX_ABI libc++ CACHE STRING "")
set(BOOTSTRAP_SANITIZER_TEST_CXX libc++ CACHE STRING "")
set(BOOTSTRAP_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
set(BOOTSTRAP_LIBCXX_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_LLVM_UNWINDER YES CACHE BOOL "")

set(BOOTSTRAP_LLVM_INSTALL_BINUTILS_SYMLINKS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_CCTOOLS_SYMLINKS ON CACHE BOOL "")

set(BOOTSTRAP_LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")

set(BOOTSTRAP_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
set(BOOTSTRAP_CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG " CACHE STRING "")
set(BOOTSTRAP_CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(BOOTSTRAP_CMAKE_ASM_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")

$ find . -name \*-tblgen
./tools/clang/stage2-bins/bin/llvm-tblgen
./bin/llvm-tblgen
./bin/clang-tblgen

FAILED: include/llvm/IR/IntrinsicsR600.h /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include/llvm/IR/IntrinsicsR600.h 
cd /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins && /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-tblgen -gen-intrinsic-enums -intrinsic-prefix=r600 -I /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include -I/home/user/Documents/src/llvm/llvm-project/llvm/include /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR/Intrinsics.td --write-if-changed -o include/llvm/IR/IntrinsicsR600.h -d include/llvm/IR/IntrinsicsR600.h.d
/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory

[arcivanov]

https://reviews.llvm.org/D118964

[arcivanov]

Actually the same thing happens with lldb-tblgen once you get to it.

[12/15] Performing build step for 'stage2'
[3220/4875] Building CommandOptions.inc...
FAILED: tools/lldb/source/Commands/CommandOptions.inc /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/source/Commands/CommandOptions.inc 
cd /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins && /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/lldb-tblgen -gen-lldb-option-defs -I /home/user/Documents/src/llvm/llvm-project/lldb/source/Commands -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/source -I/home/user/Documents/src/llvm/llvm-project/lldb/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include -I/home/user/Documents/src/llvm/llvm-project/llvm/include -I/home/user/.pyenv/versions/3.9.9/include/python3.9 -I/home/user/Documents/src/llvm/llvm-project/clang/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/../clang/include -I/usr/include -I/usr/include/libxml2 -I/home/user/Documents/src/llvm/llvm-project/lldb/source/. /home/user/Documents/src/llvm/llvm-project/lldb/source/Commands/Options.td --write-if-changed -o tools/lldb/source/Commands/CommandOptions.inc -d tools/lldb/source/Commands/CommandOptions.inc.d
/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/lldb-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory

[arcivanov]

And

/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/clang-ast-dump: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory

But this can't be overwritten with -DXYZ_TABLEGEN=<path>, so now another solution has to be found.

[jonhoo]

For future reference, I ended up fixing this by building stage1 with

LDFLAGS="-Wl,-rpath=. -Wl,--disable-new-dtags"

which causes all the stage1 binaries to encode the path to each library they link against into the binary itself using RPATH (as suggested briefly here). The second flag is specifically so that RPATH is set rather than RUNPATH, because the former takes precedence over LD_LIBRARY_PATH. This in turn means you can then run the stage2 build with LD_LIBRARY_PATH set. Specifically:

LD_LIBRARY_PATH="$stage_2_build/lib:$stage_1_install/lib:$LD_LIBRARY_PATH" make

[arcivanov]

@jonhoo Let me see if it works in my scenario. Thanks so much!!

[arcivanov]

Ah, you're using make. I need to find a similar way to do it with Ninja.

[jonhoo]

Also, to leave searchable breadcrumbs for others who may be running into this issue, it's also possible you run into this issue in the form of an error like:

[ 95%] ASTNodeAPI.json
../../../../bin/clang-ast-dump: relocation error: ../../../../bin/clang-ast-dump: symbol _ZTVN4llvm2cl6parserINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEEEE, version LLVM_14 not defined in file libLLVM-14.so with link time reference

or this one:

../../../../bin/llvm-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory

There is also some additional discussion in https://discourse.llvm.org/t/bootstrap-build-with-llvm-runtimes-clang-shared-libc-build-failure-stage2-bins-due-to-libc-so-not-found/60141

[jonhoo]

Ah, you're using make. I need to find a similar way to do it with Ninja.

I think the exact same thing should work for ninja, just s/make/ninja/ in that command.

[arcivanov]

Thanks. I wonder if I can convert it into a cmake-based solution.

[arcivanov]

Thanks. With some caveats I was able to use LDFLAGS/LD_LIBRARY_PATH approach, although it required fixing #54955. I'll post the workup once I'm done with #54955