diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ef8de5a2..74c2d8a0 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -72,6 +72,7 @@ jobs:
       - name: Restore cached PLTs
         uses: actions/cache@v4
         id: plt_cache
+        if: ${{ matrix.dialyzer }}
         with:
           key: |
             ${{ runner.os }}-${{ matrix.elixir }}-${{ matrix.erlang }}-plt
@@ -82,7 +83,7 @@ jobs:
 
       # Create PLTs if no cached PLTs were found
       - name: Create PLTs
-        if: steps.plt_cache.outputs.cache-hit != 'true'
+        if: ${{ matrix.dialyzer }} && steps.plt_cache.outputs.cache-hit != 'true'
         run: MIX_ENV=test mix dialyzer --plt
 
       - name: Run dialyzer