diff --git a/lib/bundler/cli/exec.rb b/lib/bundler/cli/exec.rb
index 865ac20248e..9162d54913c 100644
--- a/lib/bundler/cli/exec.rb
+++ b/lib/bundler/cli/exec.rb
@@ -21,7 +21,7 @@ def run
       validate_cmd!
       SharedHelpers.set_bundle_environment
       if bin_path = Bundler.which(cmd)
-        kernel_load(bin_path, *args) && return if ruby_shebang?(bin_path)
+        kernel_load(bin_path, *args) if ruby_shebang?(bin_path)
         # First, try to exec directly to something in PATH
         kernel_exec([bin_path, cmd], *args)
       else
@@ -61,6 +61,7 @@ def kernel_load(file, *args)
       Bundler.ui = nil
       require "bundler/setup"
       Kernel.load(file)
+      exit
     rescue SystemExit
       raise
     rescue Exception => e # rubocop:disable Lint/RescueException
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index 4e5ca7d1f8d..8a0fdbaeeb2 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -361,25 +361,31 @@ def stub_source_index(specs)
     # +specs+
     def replace_bin_path(specs)
       gem_class = (class << Gem; self; end)
-      redefine_method(gem_class, :bin_path) do |name, *args|
-        exec_name = args.first
-
-        return ENV["BUNDLE_BIN_PATH"] if exec_name == "bundle"
 
-        spec = nil
+      redefine_method(gem_class, :find_spec_for_exe) do |name, *args|
+        exec_name = args.first
 
-        if exec_name
-          spec = specs.find {|s| s.executables.include?(exec_name) }
-          raise(Gem::Exception, "can't find executable #{exec_name}") unless spec
-          unless spec.name == name
-            warn "Bundler is using a binstub that was created for a different gem.\n" \
-              "This is deprecated, in future versions you may need to `bundle binstub #{name}` " \
-              "to work around a system/bundle conflict."
-          end
+        spec = if exec_name
+          specs.find {|s| s.executables.include?(exec_name) }
         else
-          spec = specs.find {|s| s.name == name }
-          raise Gem::Exception, "no default executable for #{spec.full_name}" unless exec_name = spec.default_executable
+          specs.find {|s| s.name == name }
+        end
+        raise(Gem::Exception, "can't find executable #{exec_name}") unless spec
+        raise Gem::Exception, "no default executable for #{spec.full_name}" unless exec_name ||= spec.default_executable
+        unless spec.name == name
+          warn "Bundler is using a binstub that was created for a different gem.\n" \
+            "This is deprecated, in future versions you may need to `bundle binstub #{name}` " \
+            "to work around a system/bundle conflict."
         end
+        spec
+      end
+
+      redefine_method(gem_class, :bin_path) do |name, *args|
+        exec_name = args.first
+        return ENV["BUNDLE_BIN_PATH"] if exec_name == "bundle"
+
+        spec = find_spec_for_exe(name, *args)
+        exec_name ||= spec.default_executable
 
         gem_bin = File.join(spec.full_gem_path, spec.bindir, exec_name)
         gem_from_path_bin = File.join(File.dirname(spec.loaded_from), spec.bindir, exec_name)
diff --git a/spec/commands/exec_spec.rb b/spec/commands/exec_spec.rb
index be2f8aa2a50..29f5927c14f 100644
--- a/spec/commands/exec_spec.rb
+++ b/spec/commands/exec_spec.rb
@@ -128,13 +128,13 @@
       G
     end
 
-    bundle "exec rackup", :expect_err => true
+    bundle! "exec rackup", :expect_err => true
 
     expect(out).to eq("0.9.1")
     expect(err).to match("deprecated")
 
     Dir.chdir bundled_app2 do
-      bundle "exec rackup"
+      bundle! "exec rackup"
       expect(out).to eq("1.0.0")
     end
   end