diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
index 544422b35..7f2ea40cf 100644
--- a/.github/workflows/continuous-integration.yml
+++ b/.github/workflows/continuous-integration.yml
@@ -56,9 +56,5 @@ jobs:
       run: |
         nix flake check --print-build-logs
 
-    - name: Compile and typecheck benchmarks
-      run: |
-        nix develop
-        cargo bench --no-run
-
-        find benches -type f -name "*.ncl" -print0 | xargs -0 -I file nix run . -- -f file typecheck
+    - name: Typecheck benchmarks
+      run: find benches -type f -name "*.ncl" -print0 | xargs -0 -I file nix run . -- -f file typecheck
diff --git a/flake.nix b/flake.nix
index 670e22809..6c6087545 100644
--- a/flake.nix
+++ b/flake.nix
@@ -182,7 +182,7 @@
 
       # Given a rust toolchain, provide Nickel's Rust dependencies, Nickel, as
       # well as rust tools (like clippy)
-      mkCraneArtifacts = { rust ? mkRust { } }:
+      mkCraneArtifacts = { rust ? mkRust { }, noRunBench ? false }:
         let
           craneLib = crane.lib.${system}.overrideToolchain rust;
 
@@ -217,6 +217,18 @@
           nickel = buildPackage "nickel-lang";
           lsp-nls = buildPackage "nickel-lang-lsp";
 
+          benchmarks = craneLib.mkCargoDerivation {
+            inherit src cargoArtifacts;
+
+            pnameSuffix = "-bench";
+
+            buildPhaseCargoCommand = ''
+              cargo bench ${pkgs.lib.optionalString noRunBench "--no-run"}
+            '';
+
+            doInstallCargoArtifacts = false;
+          };
+
           rustfmt = craneLib.cargoFmt {
             # Notice that unlike other Crane derivations, we do not pass `cargoArtifacts` to `cargoFmt`, because it does not need access to dependencies to format the code.
             inherit src;
@@ -380,6 +392,7 @@
       packages = {
         inherit (mkCraneArtifacts { })
           nickel
+          benchmarks
           lsp-nls;
         default = pkgs.buildEnv {
           name = "nickel";
@@ -407,10 +420,11 @@
       };
 
       checks = {
-        inherit (mkCraneArtifacts { })
-          nickel
-          lsp-nls
+        inherit (mkCraneArtifacts { noRunBench = true; })
+          benchmarks
           clippy
+          lsp-nls
+          nickel
           rustfmt;
         # An optimizing release build is long: eschew optimizations in checks by
         # building a dev profile