diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8b7f20dac..5e2ae545d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,7 +5,7 @@ steps:
           version: "1.6"
       - JuliaCI/julia-test#v1: ~
     command:
-      - mkdir -p "${JULIA_DEPOT_PATH}/conda/3/x86_64"
+      - mkdir -p "$${JULIA_DEPOT_PATH}/conda/3/x86_64"
     agents:
       queue: "juliagpu"
       cuda: "*"
@@ -17,7 +17,7 @@ steps:
           version: "1"
       - JuliaCI/julia-test#v1: ~
     command:
-      - mkdir -p "${JULIA_DEPOT_PATH}/conda/3/x86_64"
+      - mkdir -p "$${JULIA_DEPOT_PATH}/conda/3/x86_64"
     agents:
       queue: "juliagpu"
       cuda: "*"
diff --git a/Project.toml b/Project.toml
index 64b79bca0..ee5a7b371 100644
--- a/Project.toml
+++ b/Project.toml
@@ -61,6 +61,7 @@ julia = "1.6"
 
 [extras]
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a"
 Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
@@ -71,4 +72,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 
 [targets]
-test = ["ChainRulesTestUtils", "CUDA", "Distances", "FFTW", "FiniteDifferences", "PyCall", "Test"]
+test = ["ChainRulesTestUtils", "Conda", "CUDA", "Distances", "FFTW", "FiniteDifferences", "PyCall", "Test"]
diff --git a/test/features.jl b/test/features.jl
index 77a9b7165..78f496721 100644
--- a/test/features.jl
+++ b/test/features.jl
@@ -682,6 +682,10 @@ end
 end
 
 @testset "PyCall custom @adjoint" begin
+  # Trigger Python install if required. Required for Buildkite CI!
+  import Conda
+  Conda.list()
+
   import PyCall
   math = PyCall.pyimport("math")
   pysin(x) = math.sin(x)