diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml
index 16be0570f..551046175 100644
--- a/script/app-mlperf-inference/meta.yaml
+++ b/script/app-mlperf-inference/meta.yaml
@@ -1740,7 +1740,7 @@ variations:
 
   cuda,reference:
     docker:
-      base_image: nvcr.io/nvidia/pytorch:24.03-py3
+      base_image: nvcr.io/nvidia/pytorch:24.08-py3
 
   cuda:
     docker:
diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py
index 5b34a3f6c..987f7ba6f 100644
--- a/script/build-dockerfile/customize.py
+++ b/script/build-dockerfile/customize.py
@@ -227,7 +227,7 @@ def preprocess(i):
     if env.get('MLC_DOCKER_EXTRA_SYS_DEPS', '') != '':
         f.write('RUN ' + env['MLC_DOCKER_EXTRA_SYS_DEPS'] + EOL)
 
-    if env['MLC_DOCKER_OS'] == "ubuntu":
+    if env['MLC_DOCKER_OS'] == "ubuntu" and False:  # Turning this off as we are using venv
         if int(str(env['MLC_DOCKER_OS_VERSION']).split('.')[0]) >= 23:
             if "--break-system-packages" not in env.get(
                     'MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''):