Skip to content

[TensorRT EP] support TensorRT 10.10-GA #24619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/linux_tensorrt_ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
build_config: Release
architecture: x64
dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
docker_build_args: '--build-arg BASEIMAGE=onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20250124.1 --build-arg TRT_VERSION=10.9.0.34-1.cuda12.8 --network=host'
docker_build_args: '--build-arg BASEIMAGE=onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20250124.1 --build-arg TRT_VERSION=10.10.0.31-1.cuda12.9 --network=host'
docker_image_repo: onnxruntimetensorrt86gpubuild
extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel --cuda_version=12.2 --cuda_home=/usr/local/cuda-12.2 --cudnn_home=/usr/local/cuda-12.2 --use_tensorrt --tensorrt_home /usr --build_java --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=80 onnxruntime_BUILD_UNIT_TESTS=ON onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON'
python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:$PATH'
Expand Down Expand Up @@ -57,7 +57,7 @@ jobs:
with:
dockerfile: ${{ github.workspace }}/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
image-name: ghcr.io/microsoft/onnxruntime/onnxruntimetensorrt86gpubuild
build-args: '--build-arg BASEIMAGE=onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20250124.1 --build-arg TRT_VERSION=10.9.0.34-1.cuda12.8 --network=host'
build-args: '--build-arg BASEIMAGE=onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20250124.1 --build-arg TRT_VERSION=10.10.0.31-1.cuda12.9 --network=host'
push: true
azure-container-registry-name: onnxruntimebuildcache
env:
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/windows_tensorrt.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ jobs:
AZCOPY_AUTO_LOGIN_TYPE: MSI
AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4

- name: Download TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8
run: 'azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/local/TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" ${{ runner.temp }}'
- name: Download TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9
run: 'azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/local/TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9" ${{ runner.temp }}'
shell: pwsh
env:
AZCOPY_AUTO_LOGIN_TYPE: MSI
Expand All @@ -67,18 +67,18 @@ jobs:
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.2\bin"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\bin"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\extras\CUPTI\lib64"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8\lib"
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9\lib"

- name: Generate sln
working-directory: ${{ runner.temp }}
run: |
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
shell: cmd

- name: Build
working-directory: ${{ runner.temp }}
run: |
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --build --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --build --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
shell: cmd

- name: Add build dir to PATH
Expand All @@ -96,6 +96,6 @@ jobs:
working-directory: ${{ runner.temp }}
run: |
mklink /D /J ${{ github.workspace }}\RelWithDebInfo\models ${{ github.workspace }}\models
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
shell: cmd
timeout-minutes: 180
4 changes: 2 additions & 2 deletions cmake/deps.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.z
mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.82.0.zip;9bc9e01dffb64d9e0773b2e44d2f22c51aace063
onnx;https://github.com/onnx/onnx/archive/7fc2b81a275223f5b02a522d9d2649837542a7be.zip;555338a12903941bb45f57540476244f9ffee17b
# Use the latest commit of 10.9-GA
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/d5dce67db7c2e64b07e055571f5ec06f7f254de2.zip;01114d3b67650857281fa50faa2e412130a63b69
# Use the latest commit of 10.10-GA
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/3b9c961a4318cea6fa1fa5f064562064eb27a9bd.zip;b668d3b25b386b4de9b7a50f7a0172897cd49cdc
protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa
protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a
protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import sys

TRT_DOCKER_FILES = {
"10.9_cuda12.8_cudnn9": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10",
"10.10_cuda12.9_cudnn9": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10",
"BIN": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt_bin",
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ variables:
- name: docker_base_image
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20250124.1
- name: linux_trt_version
value: 10.9.0.34-1.cuda12.8
value: 10.10.0.31-1.cuda12.9
- name: Repository
value: 'onnxruntimecuda12manylinuxbuild'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ parameters:
- name: TrtVersion
displayName: TensorRT Version
type: string
default: 10.9_cuda12.8_cudnn9
default: 10.10_cuda12.9_cudnn9
values:
- 10.9_cuda12.8_cudnn9
- 10.10_cuda12.9_cudnn9
- BIN

- name: UseTensorrtOssParser
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
variables:
common_trt_version: '10.9.0.34'
common_trt_version: '10.10.0.31'
# As for Debian installation, replace '-1.' by '-1+' when assigning trt version below
linux_trt_version_cuda11: ${{ variables.common_trt_version }}-1.cuda11.8
linux_trt_version_cuda12: ${{ variables.common_trt_version }}-1.cuda12.8
linux_trt_version_cuda12: ${{ variables.common_trt_version }}-1.cuda12.9
win_trt_folder_cuda11: TensorRT-${{ variables.common_trt_version }}.Windows10.x86_64.cuda-11.8
win_trt_folder_cuda12: TensorRT-${{ variables.common_trt_version }}.Windows10.x86_64.cuda-12.8
win_trt_folder_cuda12: TensorRT-${{ variables.common_trt_version }}.Windows10.x86_64.cuda-12.9
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ parameters:
- 12.2
- name: TrtVersion
type: string
default: '10.9.0.34'
default: '10.10.0.31'
values:
- 8.6.1.6
- 10.9.0.34
- 10.10.0.31

steps:
- ${{ if eq(parameters.DownloadCUDA, true) }}:
Expand All @@ -42,7 +42,7 @@ steps:
- powershell: |
Write-Host "##vso[task.setvariable variable=trtCudaVersion;]12.0"
displayName: Set trtCudaVersion
- ${{ if and(eq(parameters.CudaVersion, '12.2'), eq(parameters.TrtVersion, '10.9.0.34')) }}:
- ${{ if and(eq(parameters.CudaVersion, '12.2'), eq(parameters.TrtVersion, '10.10.0.31')) }}:
- powershell: |
Write-Host "##vso[task.setvariable variable=trtCudaVersion;]12.8"
displayName: Set trtCudaVersion
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ parameters:
# default: '11.8'
# - name: win_trt_folder_cuda11
# type: string
# default: 'TensorRT-10.9.0.34.Windows10.x86_64.cuda-11.8'
# default: 'TensorRT-10.10.0.31.Windows10.x86_64.cuda-11.8'
- name: win_trt_folder_cuda12
type: string
default: 'TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8'
default: 'TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9'

steps:
- ${{ if eq(parameters.DownloadCUDA, 'true') }}:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
ARG BASEIMAGE=nvidia/cuda:12.5.1-cudnn-devel-ubi8

FROM $BASEIMAGE
ARG TRT_VERSION=10.9.0.34-1.cuda12.8
ARG TRT_VERSION=10.10.0.31-1.cuda12.9

#Install TensorRT only if TRT_VERSION is not empty
RUN if [ -n "$TRT_VERSION" ]; then \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# Build base image with required system packages
ARG BASEIMAGE=nvidia/cuda:12.5.1-cudnn-devel-ubi8
ARG TRT_VERSION=10.9.0.34-1.cuda12.8
ARG TRT_VERSION=10.10.0.31-1.cuda12.9
FROM $BASEIMAGE AS base
ARG TRT_VERSION
ENV PATH=/opt/python/cp310-cp310/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${PATH}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# Build base image with required system packages
ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubi8
ARG TRT_VERSION=10.9.0.34-1.cuda11.8
ARG TRT_VERSION=10.10.0.31-1.cuda11.8
FROM $BASEIMAGE AS base
ARG TRT_VERSION
ENV PATH=/opt/python/cp310-cp310/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${PATH}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# Build base image with required system packages
ARG BASEIMAGE=nvidia/cuda:12.2.2-cudnn8-devel-ubuntu20.04
ARG TRT_VERSION=10.9.0.34-1+cuda12.8
ARG TRT_VERSION=10.10.0.31-1+cuda12.9
ARG LD_LIBRARY_PATH_ARG=/usr/local/lib64:/usr/local/cuda/lib64
FROM $BASEIMAGE AS base
ARG TRT_VERSION
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# Build base image with required system packages
ARG BASEIMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
ARG TRT_VERSION=10.9.0.34-1+cuda12.8
ARG TRT_VERSION=10.10.0.31-1+cuda12.9
ARG LD_LIBRARY_PATH_ARG=/usr/local/lib64:/usr/local/cuda/lib64
FROM $BASEIMAGE AS base
ARG TRT_VERSION
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# Build base image with required system packages
ARG BASEIMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
ARG TRT_VERSION=10.9.0.34-1+cuda12.8
ARG TRT_VERSION=10.10.0.31-1+cuda12.9
ARG LD_LIBRARY_PATH_ARG=/usr/local/lib64:/usr/local/cuda/lib64
FROM $BASEIMAGE AS base
ARG TRT_VERSION
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ RUN pip install --upgrade pip
RUN pip install setuptools>=68.2.2 psutil

# Install TensorRT
RUN TRT_VERSION="10.9.0.34-1+cuda12.8" &&\
RUN TRT_VERSION="10.10.0.31-1+cuda12.9" &&\
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\
apt-get update &&\
apt-get install -y \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
ARG BASEIMAGE=nvidia/cuda:12.5.1-cudnn-devel-ubi8

FROM $BASEIMAGE
ARG TRT_VERSION=10.9.0.34-1.cuda12.8
ARG TRT_VERSION=10.10.0.31-1.cuda12.9

#Install TensorRT only if TRT_VERSION is not empty
RUN if [ -n "${TRT_VERSION}" ]; then \
Expand Down
4 changes: 2 additions & 2 deletions tools/ci_build/github/windows/setup_env_gpu.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v12.2\ (
) else (
set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\CUPTI\lib64;%PATH%
)
set PATH=%AGENT_TEMPDIRECTORY%\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8\lib;%PATH%
set PATH=%AGENT_TEMPDIRECTORY%\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9\lib;%PATH%

@REM The default version is still cuda v12.2, because set cuda v11.8 after it
set PATH=%PATH%;%AGENT_TEMPDIRECTORY%\TensorRT-10.9.0.34.Windows10.x86_64.cuda-11.8\lib
set PATH=%PATH%;%AGENT_TEMPDIRECTORY%\TensorRT-10.10.0.31.Windows10.x86_64.cuda-11.8\lib
if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ (
set PATH=%PATH%;%AGENT_TEMPDIRECTORY%\v11.8\bin;%AGENT_TEMPDIRECTORY%\v11.8\extras\CUPTI\lib64
) else (
Expand Down
2 changes: 1 addition & 1 deletion tools/ci_build/github/windows/setup_env_trt.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v12.2\ (
) else (
set PATH=%PATH%;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\CUPTI\lib64
)
set PATH=%AGENT_TEMPDIRECTORY%\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8\lib;%PATH%
set PATH=%AGENT_TEMPDIRECTORY%\TensorRT-10.10.0.31.Windows10.x86_64.cuda-12.9\lib;%PATH%
set GRADLE_OPTS=-Dorg.gradle.daemon=false
set CUDA_MODULE_LOADING=LAZY
Loading