diff --git a/tensorflow_serving/tools/docker/Dockerfile.gpu b/tensorflow_serving/tools/docker/Dockerfile.gpu
index 88b0410fc34..13a4222d910 100644
--- a/tensorflow_serving/tools/docker/Dockerfile.gpu
+++ b/tensorflow_serving/tools/docker/Dockerfile.gpu
@@ -49,7 +49,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
         libfreetype6-dev \
         pkg-config \
         software-properties-common \
-        unzip
+        unzip \
+        cuda-compiler-11-8
 
 # We don't install libnvinfer-dev since we don't need to build against TensorRT
 RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub && \