diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml
index 67c7befb31dc..7aab5ae959ea 100644
--- a/.github/workflows/hvd-tests.yml
+++ b/.github/workflows/hvd-tests.yml
@@ -23,15 +23,37 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+
+      - name: Get year & week number
+        id: get-date
+        run: echo "::set-output name=date::$(/bin/date "+%Y-%U")"
+        shell: bash -l {0}
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          pip3 install -U --user pip
+          echo "::set-output name=dir::$(pip3 cache dir)"
+        shell: bash -l {0}
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.get-date.outputs.date }}-horovod-${{ hashFiles('requirements-dev.txt') }}
+
       - name: Setup Miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
       - name: Install dependencies
         shell: bash -l {0}
         run: |
+
           #install other dependencies
           conda install pytorch torchvision cpuonly -c ${{ matrix.pytorch-channel }}
           pip install -r requirements-dev.txt
diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml
index 91ac039549d4..9d0abff331fa 100644
--- a/.github/workflows/pytorch-version-tests.yml
+++ b/.github/workflows/pytorch-version-tests.yml
@@ -1,3 +1,5 @@
+name: PyTorch version tests
+
 on:
   # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule
   schedule:
@@ -22,11 +24,34 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get year & week number
+        id: get-date
+        run: echo "::set-output name=date::$(/bin/date "+%Y-%U")"
+        shell: bash -l {0}
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          pip3 install -U --user pip
+          echo "::set-output name=dir::$(pip3 cache dir)"
+        shell: bash -l {0}
+
+      - uses: actions/cache@v2
+        shell: bash -l {0}
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-
+
       - name: Setup Miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
       - name: Install dependencies
         shell: bash -l {0}
diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml
index 2e07cf2e8712..9bd9df3bf71f 100644
--- a/.github/workflows/tpu-tests.yml
+++ b/.github/workflows/tpu-tests.yml
@@ -29,6 +29,26 @@ jobs:
           python-version: "3.6.10"
           architecture: "x64"
 
+      - name: Get year & week number
+        id: get-date
+        run: echo "::set-output name=date::$(/bin/date "+%Y-%U")"
+        shell: bash -l {0}
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          pip3 install -U pip
+          echo "::set-output name=dir::$(pip cache dir)"
+        shell: bash -l {0}
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.xla-version }}-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.xla-version }}-
+
       - name: Install Torch XLA and others
         run: |
 
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index f4a56ddb0974..ffa915a2621a 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -43,11 +43,35 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+
+      - name: Get year & week number
+        id: get-date
+        run: |
+          echo "::set-output name=date::$(/bin/date "+%Y-%U")"
+        shell: bash -l {0}
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          pip3 install -U --user pip
+          echo "::set-output name=dir::$(pip3 cache dir)"
+        shell: bash -l {0}
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-
+
       - name: Setup Miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
       - name: Install dependencies
         shell: bash -l {0}