diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 50ee6354b..e9dce87bf 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -171,6 +171,9 @@ jobs:
           retention-days: 7
 
   build-wheels:
+    env:
+      # Skip rebuilding the CPU library when building the wheels.
+      BNB_SKIP_CMAKE: 1
     needs:
       - build-cpu
       - build-cuda
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2b8d0b03d..981dfdc77 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -10,6 +10,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  # Skip rebuilding the CPU library when installing the wheels.
+  # We build the libraries in separate jobs and upload as artifacts.
+  BNB_SKIP_CMAKE: 1
+
 jobs:
 
   build-cpu:
@@ -146,7 +151,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install torch==${{ matrix.torch_version }} --index-url https://download.pytorch.org/whl/cpu
-          pip install -e ".[test]"
+          pip install -e ".[test]" -v
           pip install pytest-cov
 
       # We need to downgrade to numpy<2 for torch<2.4.1 compatibility on Windows
@@ -188,7 +193,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cpu
-          pip install -e ".[test]"
+          pip install -e ".[test]" -v
           pip install pytest-cov
 
       - name: Show installed packages
@@ -263,7 +268,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e ".[test]"
+          pip install -e ".[test]" -v
           pip install pytest-cov
 
       - name: Show installed packages
@@ -321,7 +326,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e ".[test]"
+          pip install -e ".[test]" -v
           pip install pytest-cov
 
       - name: Show installed packages
@@ -438,7 +443,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install --pre torch~=${{ matrix.torch_version }}.dev0 --index-url ${{ matrix.pypi_index }}
-          pip install -e ".[test]"
+          pip install -e ".[test]" -v
           pip install pytest-cov
       - name: Show installed packages
         run: pip list
diff --git a/pyproject.toml b/pyproject.toml
index 748b77d90..4f807e04e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -163,3 +163,8 @@ ignore_missing_imports = true
 [[tool.mypy.overrides]]
 module = "scipy.stats"
 ignore_missing_imports = true
+
+[tool.scikit-build]
+cmake.build-type = "Release"
+cmake.build-args = ["--config", "Release"]
+wheel.cmake = false
diff --git a/setup.py b/setup.py
index fc1b99929..4661db90c 100644
--- a/setup.py
+++ b/setup.py
@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 from distutils.errors import DistutilsModuleError
+import os
 from warnings import warn
 
 from setuptools import find_packages, setup
@@ -18,24 +19,31 @@ def has_ext_modules(self):
 
 class ExtBuildPy(build_py):
     def run(self):
-        # build_cmake needs to be called prior to build_py, as the latter
-        # collects the files output into the package directory.
-        try:
-            self.run_command("build_cmake")
-        except DistutilsModuleError:
-            warn(
-                "scikit-build-core not installed, CMake will not be invoked automatically. "
-                "Please install scikit-build-core or run CMake manually to build extensions."
-            )
+        if os.environ.get("BNB_SKIP_CMAKE", "").lower() in ("1", "true", "yes"):
+            print("skipping CMake build")
+        else:
+            # build_cmake needs to be called prior to build_py, as the latter
+            # collects the files output into the package directory.
+            try:
+                self.run_command("build_cmake")
+            except DistutilsModuleError:
+                warn(
+                    "scikit-build-core not installed, CMake will not be invoked automatically. "
+                    "Please install scikit-build-core or run CMake manually to build extensions."
+                )
         super().run()
 
 
-setup(
-    version="0.49.0.dev0",
-    packages=find_packages(),
-    distclass=BinaryDistribution,
-    cmake_source_dir=".",
-    cmdclass={
-        "build_py": ExtBuildPy,
-    },
-)
+cmdclass = {"build_py": ExtBuildPy}
+
+setup_kwargs = {
+    "version": "0.49.0.dev0",
+    "packages": find_packages(),
+    "distclass": BinaryDistribution,
+    "cmdclass": {"build_py": ExtBuildPy},
+}
+
+if os.environ.get("BNB_SKIP_CMAKE", "").lower() not in ("1", "true", "yes"):
+    setup_kwargs["cmake_source_dir"] = "."
+
+setup(**setup_kwargs)