diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml
index c96981c5609..c475f9b2110 100644
--- a/.github/workflows/cli-tests.yaml
+++ b/.github/workflows/cli-tests.yaml
@@ -61,10 +61,29 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
+
+      # We can remove this once we fix nix golangci-lint issue and move to devbox
       - uses: actions/setup-go@v4
         with:
           go-version-file: ./go.mod
-      - run:  go run ./cmd/devbox run fmt
+          cache: false
+
+      - name: Install devbox
+        uses: jetpack-io/devbox-install-action@v0.7.0
+        with:
+          enable-cache: true
+
+      - name: Mount golang cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/golangci-lint
+            ~/.cache/go-build
+            ~/go/pkg
+          key: go-${{ runner.os }}-${{ hashFiles('go.sum') }}
+
+      - run:  devbox run fmt
+      
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3.7.0
         with:
diff --git a/devbox.json b/devbox.json
index 9e224de0a45..15d0d0f50d5 100644
--- a/devbox.json
+++ b/devbox.json
@@ -21,23 +21,7 @@
       ],
       "code": "code .",
       "lint": "golangci-lint run",
-      "fmt": [
-        "mkdir -p dist/tools",
-        "export GOBIN=\"$PWD/dist/tools\"",
-        "go install mvdan.cc/gofumpt@latest",
-        "if [ -n \"${CI:-}\" ]; then",
-        "    unformatted=\"$(find . -name '*.go' -exec \"$GOBIN/gofumpt\" -l {} \\+ | awk '{ print \"\t\" $0 }')\"",
-        "    if [ -n \"$unformatted\" ]; then",
-        "        echo 'The following files need to be formatted with gofumpt:'",
-        "        echo",
-        "        echo \"$unformatted\"",
-        "        echo",
-        "        echo 'Run `devbox run fmt` to format them.'",
-        "        exit 1",
-        "    fi",
-        "fi",
-        "find . -name '*.go' -exec \"$GOBIN/gofumpt\" -extra -w {} \\+"
-      ],
+      "fmt": "scripts/gofumpt.sh",
       "test": "go test -race -cover ./...",
       "update-examples": "devbox run build && go run testscripts/testrunner/updater/main.go"
     }
diff --git a/scripts/gofumpt.sh b/scripts/gofumpt.sh
new file mode 100755
index 00000000000..69e44a0a1cc
--- /dev/null
+++ b/scripts/gofumpt.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+mkdir -p dist/tools
+export GOBIN="$PWD/dist/tools"
+go install mvdan.cc/gofumpt@latest
+
+find . -name '*.go' -exec "$GOBIN/gofumpt" -extra -w {} \+
+
+if [ -n "${CI:-}" ]; then
+  git diff --exit-code
+fi