diff --git a/.github/workflows/unit-test-optimize.yml b/.github/workflows/unit-test-optimize.yml
index 717a8769718..d210c6a26c3 100644
--- a/.github/workflows/unit-test-optimize.yml
+++ b/.github/workflows/unit-test-optimize.yml
@@ -43,6 +43,7 @@ jobs:
             test_name: "PR-test"
           - test_branch: "main"
             test_name: "baseline"
+      fail-fast: true
     name: optimize-unit-test-${{ matrix.test_name }}
     steps:
       - name: Docker Clean Up
diff --git a/tests/requirements.txt b/tests/requirements.txt
index aa30ade21ce..d82e58894d4 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -16,7 +16,7 @@ mlflow
 nlpaug==1.1.9
 onnx
 onnxruntime
-optimum
+optimum-intel==1.16.1
 peft==0.6.2
 py-cpuinfo
 sacremoses