Implement Nitro on top of TensorRT-LLM C++ repo

.github/runners/Dockerfile.window.runner-ada

@@ -0,0 +1,291 @@
+# Use the Windows Server Core 2019 image.
+# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2022
+
+# Use the Windows Server Core 2019 image.
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
+
+# Restore the default Windows shell for correct batch processing.
+# (Used for VS Build Tools installation)
+SHELL ["cmd", "/S", "/C"]
+
+# -----------------------------------------------------------------------------
+
+# Install CUDA 12.2
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/cuda_12.2.2_537.13_windows.exe \
+    --output "cuda_installer.exe"; \
+    Start-Process cuda_installer.exe -Wait -ArgumentList '-s'; \
+    Remove-Item cuda_installer.exe -Force
+
+# -----------------------------------------------------------------------------
+
+# Install Python 3.10.11
+
+# Download and install Python
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/python-3.10.11-amd64.exe --output python-3.10.11.exe ; \
+    Start-Process python-3.10.11.exe -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' ; \
+    Remove-Item python-3.10.11.exe -Force
+
+# Add python3 command
+RUN powershell -Command \
+    cp "\"C:\\\\Program Files\\\\Python310\\\\python.exe\" \"C:\\\\Program Files\\\\Python310\\\\python3.exe\""
+
+# -----------------------------------------------------------------------------
+
+# Install Microsoft MPI
+
+# The latest version is 10.1.3, but it requires you to get a temporary download
+# link.
+# https://learn.microsoft.com/en-us/message-passing-interface/microsoft-mpi-release-notes
+# We use 10.1.1 which has a release on the GitHub page
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/msmpisetup.exe \
+    --output "msmpisetup.exe"; \
+    Start-Process .\msmpisetup.exe -Wait ; \
+    Remove-Item msmpisetup.exe -Force
+
+# Add MPI binaries to Path
+RUN setx Path "%Path%;C:\Program Files\Microsoft MPI\Bin"
+
+# Download the MSMPI SDK
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/msmpisdk.msi \
+    --output "msmpisdk.msi"; \
+    Start-Process msiexec.exe -Wait -ArgumentList '/I msmpisdk.msi /quiet'; \
+    Remove-Item msmpisdk.msi -Force
+
+# -----------------------------------------------------------------------------
+
+# Install CMake
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/cmake-3.27.7-windows-x86_64.msi \
+    --output "cmake.msi"; \
+    Start-Process msiexec.exe -Wait -ArgumentList '/I cmake.msi /quiet'; \
+    Remove-Item cmake.msi -Force
+
+# Add CMake binaries to Path
+RUN setx Path "%Path%;C:\Program Files\CMake\bin"
+
+# -----------------------------------------------------------------------------
+
+# Install VS Build Tools
+
+RUN \
+    # Download the Build Tools bootstrapper.
+    curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe \
+    \
+    # Install Build Tools with the Microsoft.VisualStudio.Workload.AzureBuildTools workload, excluding workloads and components with known issues.
+    && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache \
+        --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" \
+        --includeRecommended \
+        --add Microsoft.VisualStudio.Workload.MSBuildTools \
+        --add Microsoft.VisualStudio.Workload.VCTools \
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 \
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 \
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 \
+        --remove Microsoft.VisualStudio.Component.Windows81SDK \
+        || IF "%ERRORLEVEL%"=="3010" EXIT 0) \
+    \
+    # Cleanup
+    && del /q vs_buildtools.exe
+
+# -----------------------------------------------------------------------------
+
+# Install Vim (can delete this but it's nice to have)
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/gvim90.exe \
+    --output "install_vim.exe"; \
+    Start-Process install_vim.exe -Wait -ArgumentList '/S'; \
+    Remove-Item install_vim.exe -Force
+
+# Add Vim binaries to Path
+RUN setx Path "%Path%;C:\Program Files (x86)\Vim\vim90"
+
+# -----------------------------------------------------------------------------
+
+# Install Chocolatey
+# Chocolatey is a package manager for Windows
+# I probably could've used it to install some of the above, but I didn't...
+
+# If you try to install Chocolatey 2.0.0, it fails on .NET Framework 4.8 installation
+# https://stackoverflow.com/a/76470753
+ENV chocolateyVersion=1.4.0
+
+# https://docs.chocolatey.org/en-us/choco/setup#install-with-cmd.exe
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    powershell.exe -NoProfile -InputFormat None -ExecutionPolicy Bypass \
+    -Command "[System.Net.ServicePointManager]::SecurityProtocol = 3072; \
+    iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))" && \
+    SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
+
+# -----------------------------------------------------------------------------
+
+# Install Git via Chocolatey
+RUN powershell -Command \
+    choco install git -y
+
+# -----------------------------------------------------------------------------
+
+# Install CUDA 11.8 NVTX
+
+#RUN powershell -Command \
+#    $ErrorActionPreference = 'Stop'; \
+#    curl.exe https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe \
+#    --output "cuda_11_installer.exe"; \
+#    Start-Process cuda_11_installer.exe -Wait -ArgumentList '-s nvtx_11.8'; \
+#    Remove-Item cuda_11_installer.exe -Force
+
+# The above command-line installation method installs NVTX headers at
+# C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\include\nvtx3\
+# CMake can't find this location for some reason.
+# Instead, we just copy the older NvToolsExt version to where CMake expects.
+# This assumes NvToolsExt was installed on the host machine using the
+# CUDA 11.8 GUI installer and copied to the build context
+
+# COPY ["NvToolsExt", "C:\\\\Program Files\\\\NVIDIA Corporation\\\\NvToolsExt"]
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/NvToolsExt.zip \
+    --output NvToolsExt.zip; \
+    Expand-Archive .\NvToolsExt.zip -DestinationPath 'C:\Program Files\NVIDIA Corporation\'; \
+    Remove-Item NvToolsExt.zip -Force
+
+# -----------------------------------------------------------------------------
+
+# Create a working directory
+WORKDIR "C:\\\\workspace"
+
+# -----------------------------------------------------------------------------
+
+# Download and unzip TensorrRT 9.2.0.5 for TensorRT-LLM
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/TensorRT-9.2.0.5.Windows10.x86_64.cuda-12.2.llm.beta.zip \
+    --output TensorRT-9.2.0.5.zip; \
+    Expand-Archive .\TensorRT-9.2.0.5.zip -DestinationPath .; \
+    Remove-Item TensorRT-9.2.0.5.zip -Force
+
+# Add TensorRT libs to Path
+RUN setx Path "%Path%;C:\workspace\TensorRT-9.2.0.5\lib"
+
+# Install TensorRT Python wheel
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    pip install TensorRT-9.2.0.5\python\tensorrt-9.2.0.post12.dev5-cp310-none-win_amd64.whl
+
+# -----------------------------------------------------------------------------
+
+# Copy cuDNN into the working directory
+# This assumes cuDNN exists on the host machine in the build context
+# COPY ["cuDNN", "cuDNN"]
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    curl.exe https://delta.jan.ai/dist/windows-container-dependencies/1/cuDNN.zip \
+    --output cuDNN.zip; \
+    Expand-Archive .\cuDNN.zip -DestinationPath .; \
+    Remove-Item cuDNN.zip -Force
+
+# Add cuDNN libs and bin to Path.
+RUN setx Path "%Path%;C:\workspace\cuDNN\lib;C:\workspace\cuDNN\bin;"
+
+# -----------------------------------------------------------------------------
+
+# Define the entry point for the docker container.
+# This entry point launches the 64-bit PowerShell developer shell.
+# We need to launch with amd64 arch otherwise Powershell defaults to x86 32-bit build commands which don't jive with CUDA
+ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]
+
+# -----------------------------------------------------------------------------
+
+# Additional dependencies to build Nitro
+
+# This bellow command lt MSVC recognize cuda compiler
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    Copy-Item -Path 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\visual_studio_integration\MSBuildExtensions\*' -Destination 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Microsoft\VC\v170\BuildCustomizations'
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    Copy-Item -Path 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\visual_studio_integration\MSBuildExtensions\*' -Destination 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Microsoft\VC\v160\BuildCustomizations'
+
+
+# Set git safe directory for nitro clone dependencies
+RUN powershell -Command \
+    git config --global --add safe.directory '*'
+
+# Package for nitro compile
+RUN powershell -Command \
+    choco install pkgconfiglite --allow-empty-checksums -y
+
+RUN powershell -Command \
+    choco install Ninja -y
+
+RUN choco install 7zip -y; \
+    7z --help
+
+# Requirements to build tensorrt-llm on windows
+# COPY ./requirements-windows.txt ./tensorrt-llm-nitro/requirements-windows.txt
+# COPY ./requirements-dev-windows.txt ./tensorrt-llm-nitro/requirements-dev-windows.txt
+# RUN powershell -Command \
+#     cd tensorrt-llm-nitro; \
+#     pip install --no-cache-dir -r .\requirements-dev-windows.txt
+
+# COPY ./.git ./tensorrt-llm-nitro/.git
+
+# COPY ./3rdparty ./tensorrt-llm-nitro/3rdparty
+
+# COPY ./cpp ./tensorrt-llm-nitro/cpp
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    git clone https://github.com/janhq/nitro-tensorrt-llm.git; \
+    cd nitro-tensorrt-llm; \
+    git checkout tensorrt-llm-nitro-rel; \
+    git submodule update --init --recursive; \
+    pip install --no-cache-dir -r .\requirements-dev-windows.txt; \
+    cd cpp/tensorrt_llm/nitro; \
+    cmake -S ./nitro_deps -B ./build_deps/nitro_deps; \
+    cmake --build ./build_deps/nitro_deps --config Release
+
+RUN setx Path "%Path%;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools"
+
+RUN VsDevCmd.bat -arch=amd64 && \
+    powershell.exe -NoLogo -ExecutionPolicy Bypass "cd nitro-tensorrt-llm; python .\scripts\build_wheel.py -a '89-real' --trt_root 'C:\workspace\TensorRT-9.2.0.5\'"
+
+# # -----------------------------------------------------------------------------
+
+# Requirements to build tensorrt-llm on windows
+ARG RUNNER_VERSION=2.314.1
+
+# Define the entry point for the docker container.
+# This entry point launches the 64-bit PowerShell developer shell.
+# We need to launch with amd64 arch otherwise Powershell defaults to x86 32-bit build commands which don't jive with CUDA
+# ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]
+
+RUN powershell -Command \
+    $ErrorActionPreference = 'Stop'; \
+    Invoke-WebRequest \
+      -Uri https://github.com/actions/runner/releases/download/v$env:RUNNER_VERSION/actions-runner-win-x64-$env:RUNNER_VERSION.zip \
+      -OutFile runner.zip; \
+    Expand-Archive -Path ./runner.zip -DestinationPath ./actions-runner; \
+    Remove-Item -Path .\runner.zip; \
+    setx /M PATH $(${Env:PATH} + \";${Env:ProgramFiles}\Git\bin\")
+
+ADD runner.ps1 ./runner.ps1
+
+RUN powershell -Command New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
+
+RUN powershell -Command icacls 'C:\workspace\nitro-tensorrt-llm' /grant 'Everyone:F' /T
+
+CMD ["powershell.exe", "-ExecutionPolicy", "Unrestricted", "-File", ".\\runner.ps1"]