diff --git a/.ci/torchbench/install.sh b/.ci/torchbench/install.sh
index e4e944e3e..95cb8e775 100644
--- a/.ci/torchbench/install.sh
+++ b/.ci/torchbench/install.sh
@@ -3,4 +3,6 @@
 set -eux
 
 python3 -c "import torch; print(torch.__version__); print(torch.version.git_version)"
-python3 install.py
+# TODO (huydhn): Look for a more permanent solution once https://github.com/pytorch/pytorch/issues/167895
+# is resolved
+python3 install.py --skip stable_diffusion_text_encoder stable_diffusion_unet
diff --git a/test.py b/test.py
index bb4884b6b..7ddc1ca48 100644
--- a/test.py
+++ b/test.py
@@ -181,6 +181,9 @@ def _load_tests():
         # api, enable after PyTorch 1.13 release
         if "quantized" in path:
             continue
+        # https://github.com/pytorch/pytorch/issues/167895
+        if "stable_diffusion" in path:
+            continue
         for device in devices:
             _load_test(path, device)