[0.5.0][Bug] Release build failure

[lanking520]

The library built from the offiical container breaks

ldd -r libtriton_tensorrtllm.so
	linux-vdso.so.1 (0x00007ffc3d582000)
	libtritonserver.so => /app/inflight_batcher_llm/build/_deps/repo-core-build/libtritonserver.so (0x00007f5237d62000)
	libmpi.so.40 => /opt/hpcx/ompi/lib/libmpi.so.40 (0x00007f5237c43000)
	libnvinfer.so.9 => /usr/local/tensorrt/lib/libnvinfer.so.9 (0x00007f52282ce000)
	libnvinfer_plugin_tensorrt_llm.so.9 => /app/inflight_batcher_llm/../tensorrt_llm/cpp/build/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so.9 (0x00007f51f1463000)
	libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f51f122d000)
	libgcc_s.so.1 => /usr/lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f51f120d000)
	libc.so.6 => /usr/lib/x86_64-linux-gnu/libc.so.6 (0x00007f51f0fe5000)
	/lib64/ld-linux-x86-64.so.2 (0x00007f523d51c000)
	libopen-rte.so.40 => /opt/hpcx/ompi/lib/libopen-rte.so.40 (0x00007f51f0f26000)
	libopen-pal.so.40 => /opt/hpcx/ompi/lib/libopen-pal.so.40 (0x00007f51f0e0d000)
	libm.so.6 => /usr/lib/x86_64-linux-gnu/libm.so.6 (0x00007f51f0d26000)
	libpthread.so.0 => /usr/lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f51f0d21000)
	libdl.so.2 => /usr/lib/x86_64-linux-gnu/libdl.so.2 (0x00007f51f0d1c000)
	librt.so.1 => /usr/lib/x86_64-linux-gnu/librt.so.1 (0x00007f51f0d17000)
	libcublas.so.12 => /usr/local/cuda/lib64/libcublas.so.12 (0x00007f51ea400000)
	libcublasLt.so.12 => /usr/local/cuda/lib64/libcublasLt.so.12 (0x00007f51c7400000)
	libcuda.so.1 => /usr/local/cuda/compat/lib.real/libcuda.so.1 (0x00007f51c57c1000)
	libnccl.so.2 => /usr/lib/x86_64-linux-gnu/libnccl.so.2 (0x00007f51b4b65000)
	libz.so.1 => /usr/lib/x86_64-linux-gnu/libz.so.1 (0x00007f51f0cf9000)
undefined symbol: cudaMemcpyAsync	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMallocAsync	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamDestroy	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamWaitEvent	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphExecDestroy	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGetDeviceCount	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemsetAsync	(./libtriton_tensorrtllm.so)
undefined symbol: cudaLaunchKernel	(./libtriton_tensorrtllm.so)
undefined symbol: cudaFree	(./libtriton_tensorrtllm.so)
undefined symbol: cudaEventSynchronize	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGetDeviceProperties_v2	(./libtriton_tensorrtllm.so)
undefined symbol: cudaPeekAtLastError	(./libtriton_tensorrtllm.so)
undefined symbol: cudaFuncSetAttribute	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphExecUpdate	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGetDevice	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemset	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphUpload	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaRegisterFunction	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamEndCapture	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceEnablePeerAccess	(./libtriton_tensorrtllm.so)
undefined symbol: cudaEventRecord	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceGetAttribute	(./libtriton_tensorrtllm.so)
undefined symbol: cudaFuncGetAttributes	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphLaunch	(./libtriton_tensorrtllm.so)
undefined symbol: cudaFreeAsync	(./libtriton_tensorrtllm.so)
undefined symbol: cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags	(./libtriton_tensorrtllm.so)
undefined symbol: cudaIpcCloseMemHandle	(./libtriton_tensorrtllm.so)
undefined symbol: cudaFreeHost	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaRegisterFatBinaryEnd	(./libtriton_tensorrtllm.so)
undefined symbol: cudaIpcOpenMemHandle	(./libtriton_tensorrtllm.so)
undefined symbol: cudaEventDestroy	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphDestroy	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceCanAccessPeer	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemPoolSetAttribute	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceGetDefaultMemPool	(./libtriton_tensorrtllm.so)
undefined symbol: cudaEventCreateWithFlags	(./libtriton_tensorrtllm.so)
undefined symbol: cudaSetDevice	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemGetInfo	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceSynchronize	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaUnregisterFatBinary	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamSynchronize	(./libtriton_tensorrtllm.so)
undefined symbol: cudaPointerGetAttributes	(./libtriton_tensorrtllm.so)
undefined symbol: cudaHostAlloc	(./libtriton_tensorrtllm.so)
undefined symbol: cudaIpcGetMemHandle	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaRegisterFatBinary	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamCreateWithPriority	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaPopCallConfiguration	(./libtriton_tensorrtllm.so)
undefined symbol: cudaStreamBeginCapture	(./libtriton_tensorrtllm.so)
undefined symbol: cudaDeviceDisablePeerAccess	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMalloc	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemcpy	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGraphInstantiate	(./libtriton_tensorrtllm.so)
undefined symbol: cudaMemPoolSetAccess	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaRegisterVar	(./libtriton_tensorrtllm.so)
undefined symbol: __cudaPushCallConfiguration	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGetLastError	(./libtriton_tensorrtllm.so)
undefined symbol: cudaGetErrorString	(./libtriton_tensorrtllm.so)

Step to reproduce

TRT_LLM_VERSION=release/0.5.0

pushd /tmp

git clone https://github.com/NVIDIA/TensorRT-LLM.git -b ${TRT_LLM_VERSION} --recursive
git clone https://github.com/triton-inference-server/tensorrtllm_backend -b ${TRT_LLM_VERSION}

rm -rf tensorrtllm_backend/tensorrt_llm
mv TensorRT-LLM tensorrtllm_backend/tensorrt_llm
cd tensorrtllm_backend/tensorrt_llm

pushd /tmp/tensorrtllm_backend/tensorrt_llm/cpp/tensorrt_llm/batch_manager/x86_64-linux-gnu
rm -rf *
wget https://github.com/NVIDIA/TensorRT-LLM/raw/release/0.5.0/cpp/tensorrt_llm/batch_manager/x86_64-linux-gnu/libtensorrt_llm_batch_manager_static.a
wget https://github.com/NVIDIA/TensorRT-LLM/raw/release/0.5.0/cpp/tensorrt_llm/batch_manager/x86_64-linux-gnu/libtensorrt_llm_batch_manager_static.pre_cxx11.a
popd

cd tensorrtllm_backend
DOCKER_BUILDKIT=1 docker build -t triton_trt_llm -f dockerfile/Dockerfile.trt_llm_backend .
popd

[thisjiang]

I fix it by apply the following diff:

diff --git a/inflight_batcher_llm/CMakeLists.txt b/inflight_batcher_llm/CMakeLists.txt
index 728092c..6199b41 100644
--- a/inflight_batcher_llm/CMakeLists.txt
+++ b/inflight_batcher_llm/CMakeLists.txt
@@ -311,13 +306,13 @@ target_link_libraries(
           triton-core-serverstub # from repo-core
           triton-backend-utils # from repo-backend
           ${MPI_LIBRARIES}
+          ${CUDA_LIBRARIES}
           nvinfer
           nvinfer_plugin_tensorrt_llm)

[kaiyux]

Thank you very much! @lanking520 @thisjiang

We will fix that in the internal codebase, and update the code on GitHub.

[kaiyux]

This issue is fixed in #50, please try on latest main branch and feel free to reopen if needed.

Closing. Thanks!