diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 24c1bd60f..47b75fe7e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -115,9 +115,12 @@ jobs:
       env:
         BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/gem_from_github.gemfile
 
-    - name: which ruby, rake, bundle
+    - name: which ruby, bundle
       shell: bash
-      run: which -a ruby rake bundle
+      run: which -a ruby bundle
+    - name: which rake
+      run: which -a rake
+      if: "!startsWith(matrix.os, 'windows')"
     - name: where ruby, rake, bundle
       if: startsWith(matrix.os, 'windows')
       run: |
diff --git a/bundler.js b/bundler.js
index df27a009d..3e86c07c0 100644
--- a/bundler.js
+++ b/bundler.js
@@ -72,7 +72,7 @@ export async function installBundler(bundlerVersionInput, rubygemsInputSet, lock
 
   if (bundlerVersion === 'default') {
     if (common.isBundler2dot2Default(engine, rubyVersion)) {
-      if (common.windows && engine === 'ruby') {
+      if (common.windows && engine === 'ruby' && (common.isStableVersion(rubyVersion) || rubyVersion === 'head')) {
         // https://github.com/ruby/setup-ruby/issues/371
         console.log(`Installing latest Bundler for ${engine}-${rubyVersion} on Windows because bin/bundle does not work in bash otherwise`)
         bundlerVersion = 'latest'
diff --git a/dist/index.js b/dist/index.js
index a0f8eeaae..67d436d90 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -86,7 +86,7 @@ async function installBundler(bundlerVersionInput, rubygemsInputSet, lockFile, p
 
   if (bundlerVersion === 'default') {
     if (common.isBundler2dot2Default(engine, rubyVersion)) {
-      if (common.windows && engine === 'ruby') {
+      if (common.windows && engine === 'ruby' && (common.isStableVersion(rubyVersion) || rubyVersion === 'head')) {
         // https://github.com/ruby/setup-ruby/issues/371
         console.log(`Installing latest Bundler for ${engine}-${rubyVersion} on Windows because bin/bundle does not work in bash otherwise`)
         bundlerVersion = 'latest'