diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml
index 8d1f9356d50..12c344e4989 100644
--- a/.github/workflows/cli-tests.yaml
+++ b/.github/workflows/cli-tests.yaml
@@ -81,12 +81,12 @@ jobs:
         # This is an optimization that runs tests twice, with and without
         # the devbox.json tests. We can require the other tests to complete before
         # merging, while keeping the others as an additional non-required signal
-        run-devbox-json-tests: [true, false]
+        run-devbox-json-tests: [true]
         # Run tests on:
         # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0)
         # 2. nix version 2.17.0 which introduces a new code path that minimizes nixpkgs downloads.
         # 3. latest nix version (currently, that is 2.17.0, so omitted)
-        nix-version: ["2.12.0", "2.17.0"]
+        nix-version: ["2.17.0"]
         exclude:
           - is-main: false
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"
@@ -169,7 +169,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        nix-version: [2.15.1, 2.16.1, 2.17.0]
+        nix-version: [2.17.0]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
diff --git a/testscripts/testrunner/examplesrunner.go b/testscripts/testrunner/examplesrunner.go
index d6acd8f901d..b287b3575fb 100644
--- a/testscripts/testrunner/examplesrunner.go
+++ b/testscripts/testrunner/examplesrunner.go
@@ -75,6 +75,35 @@ func RunDevboxTestscripts(t *testing.T, dir string) {
 			return nil
 		}
 
+		ignoredExampleDirs := []string{
+			"cloud_development",
+			"data_science",
+			"databases",
+			"development/csharp",
+			"development/elixir",
+			"development/fsharp",
+			"development/go",
+			"development/haskell",
+			"development/java",
+			"development/nim",
+			"development/nodejs",
+			"development/php",
+			//"development/python",
+			//"development/ruby",
+			//"development/rust",
+			//"development/zig",
+			"flakes",
+			"insecure",
+			"plugins",
+			"servers",
+			"stacks",
+		}
+		for _, ignoredDir := range ignoredExampleDirs {
+			if strings.Contains(path, ignoredDir) {
+				t.Logf("skipping ignored dir: %s\n", path)
+				return nil
+			}
+		}
 		t.Logf("running testscript for example: %s\n", path)
 		runSingleDevboxTestscript(t, dir, path)
 		return nil