diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml
index e8b3db8ab7a..64f8d04346e 100644
--- a/.github/workflows/tests-pypi.yml
+++ b/.github/workflows/tests-pypi.yml
@@ -84,13 +84,15 @@ jobs:
 
     # This installs the stuff needed to build and install Shapely and CartoPy from source.
     # Need to install numpy first to make CartoPy happy.
-    - name: Install dependencies
+    - name: Install CartoPy build dependencies
+      if: ${{ matrix.no-extras != 'No Extras' }}
       run: |
         sudo apt-get install libgeos-dev libproj-dev proj-bin
-        python -m pip install --upgrade pip setuptools
         python -m pip install --no-binary :all: shapely
         python -m pip install -c ci/${{ matrix.dep-versions }} numpy
-        python -m pip install -r ci/test_requirements.txt -c ci/${{ matrix.dep-versions }}
+
+    - name: Install test dependencies
+      run: python -m pip install -r ci/test_requirements.txt -c ci/${{ matrix.dep-versions }}
 
     # This imports CartoPy to find its map data cache directory
     - name: Get CartoPy maps dir