diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml
new file mode 100644
index 0000000..a2658af
--- /dev/null
+++ b/.github/workflows/IntegrationTest.yml
@@ -0,0 +1,53 @@
+name: IntegrationTest
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+
+jobs:
+  test:
+    name: ${{ matrix.package.repo }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: [1]
+        os: [ubuntu-latest]
+        package:
+          - {user: JuliaDiff, repo: ChainRulesTestUtils.jl}
+          - {user: JuliaDiff, repo: ChainRules.jl}
+          - {user: JuliaMath, repo: SpecialFunctions.jl}
+          - {user: invenia, repo: BlockDiagonals.jl}
+          - {user: invenia, repo: PDMatsExtras.jl}
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia-version }}
+          arch: x64
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Clone Downstream
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
+          path: downstream
+      - name: Load this and run the downstream tests
+        shell: julia --project=downstream {0}
+        run: |
+          using Pkg
+          try
+            # force it to use this PR's version of the package
+            Pkg.develop(PackageSpec(path="."))  # ResolverError with main deps may be thrown here
+            Pkg.update()
+            Pkg.test()  # ResolverError with test deps may be thrown here (before test process starts)
+          catch err
+            err isa Pkg.Resolve.ResolverError || rethrow()
+            # If we can't resolve that means this is incompatible by SemVer and this is fine
+            # It means we marked this as a breaking change, so we don't need to worry about
+            # Mistakenly introducing a breaking change, as we have intentionally made one
+
+            @info "Not compatible with this release. No problem." exception=err
+            exit(0)  #  Exit immediately, as a success
+          end
diff --git a/.github/workflows/IntegrationTestChainRules.yml b/.github/workflows/IntegrationTestChainRules.yml
deleted file mode 100644
index 5b7d047..0000000
--- a/.github/workflows/IntegrationTestChainRules.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-name: IntegrationTestChainRules 
-on:
-  push:
-    branches: [master]
-    tags: [v*]
-  pull_request:
-
-jobs:
-  test:
-    name: Julia v${{ matrix.julia-version }} - ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        julia-version: [1]
-        os: [ubuntu-latest]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
-        with:
-          version: ${{ matrix.julia-version }}
-          arch: x64
-      - uses: julia-actions/julia-buildpkg@latest
-      - name: Clone ChainRules
-        uses: actions/checkout@v2
-        with:
-          repository: JuliaDiff/ChainRules.jl
-          path: "ChainRules.jl"
-      - name: Run the tests
-        run: julia --project="ChainRules.jl" -e "using Pkg; Pkg.develop(PackageSpec(path=\"./\")); Pkg.update(); Pkg.test()"
-        shell: bash