Merge pull request #9944 from Tishj/pyproject_toml

[Python][Dev] Switch to using `pyproject.toml` when building and running CI.

.github/workflows/Python.yml

@@ -82,7 +82,7 @@ jobs:
       run: |
         export DISTUTILS_C_COMPILER_LAUNCHER=ccache
         # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
-        cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
+        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
 
    manylinux-extensions-x64:
     # Builds extensions for linux_md64_gcc4
@@ -204,7 +204,7 @@ jobs:
         if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
           export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
         fi
-        cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
+        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
         ls wheelhouse
 
     - name: Deploy
@@ -264,7 +264,7 @@ jobs:
           if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
             export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
           fi
-          cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
+          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
 
       - name: Deploy
         env:
@@ -332,7 +332,7 @@ jobs:
           if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
             export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
           fi
-          cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
+          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
 
       - name: Deploy
         env:

.github/workflows/Regression.yml

@@ -242,7 +242,7 @@ jobs:
       shell: bash
       run: |
         cd tools/pythonpkg
-        python setup.py install --user
+        pip install --use-pep517 . --user
         cd ../..
 
     - name: Run New Version
@@ -262,7 +262,7 @@ jobs:
       run: |
         git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
         cd duckdb/tools/pythonpkg
-        python setup.py install --user
+        pip install --use-pep517 . --user
         cd ../../..
 
     - name: Run Current Version

tools/pythonpkg/MANIFEST.in

@@ -3,5 +3,4 @@ recursive-include duckdb *.h *.hpp *.cpp
 include duckdb_python.cpp
 recursive-include src *.h *.hpp *.cpp
 recursive-include duckdb-stubs *.pyi
-recursive-include duckdb
-include cibw.toml
+recursive-include duckdb

tools/pythonpkg/cibw.toml → tools/pythonpkg/pyproject.toml

@@ -1,4 +1,17 @@
-# Config for cibuildwheel, some values are overridden by env variables of specific jobs
+[build-system]
+requires = [
+	"setuptools>=60",
+	"setuptools_scm>=6.4",
+	"pybind11>=2.6.0",
+	"wheel>=0.38.0"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../.."
+local_scheme = {env = "SETUPTOOLS_SCM_LOCAL_SCHEME", default = "no-local-version"}
+
+### CI Builwheel configurations ###
 
 # Default config runs all tests and requires at least one extension to be tested against
 [tool.cibuildwheel]

tools/pythonpkg/setup.py

@@ -328,10 +328,13 @@ def setup_data_files(data_files):
     'duckdb-stubs',
     'duckdb-stubs.functional',
     'duckdb-stubs.typing',
+    'duckdb-stubs.value',
+    'duckdb-stubs.value.constant',
     'adbc_driver_duckdb',
 ]
 
 spark_packages = [
+    'duckdb.experimental',
     'duckdb.experimental.spark',
     'duckdb.experimental.spark.sql',
     'duckdb.experimental.spark.errors',
@@ -352,8 +355,6 @@ def setup_data_files(data_files):
     packages=packages,
     include_package_data=True,
     python_requires='>=3.7.0',
-    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
-    use_scm_version=setuptools_scm_conf,
     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
     classifiers=[
         'Topic :: Database :: Database Engines/Servers',

tools/pythonpkg/src/pyexpression.cpp

@@ -10,7 +10,7 @@ namespace duckdb {
 
 DuckDBPyExpression::DuckDBPyExpression(unique_ptr<ParsedExpression> expr_p, OrderType order_type,
                                        OrderByNullType null_order)
-    : expression(std::move(expr_p)), order_type(order_type), null_order(null_order) {
+    : expression(std::move(expr_p)), null_order(null_order), order_type(order_type) {
 	if (!expression) {
 		throw InternalException("DuckDBPyExpression created without an expression");
 	}
@@ -325,7 +325,7 @@ shared_ptr<DuckDBPyExpression> DuckDBPyExpression::CaseExpression(const DuckDBPy
 	// Add NULL as default Else expression
 	auto &internal_expression = reinterpret_cast<duckdb::CaseExpression &>(*case_expr->expression);
 	internal_expression.else_expr = make_uniq<duckdb::ConstantExpression>(Value(LogicalTypeId::SQLNULL));
-	return std::move(case_expr);
+	return case_expr;
 }
 
 shared_ptr<DuckDBPyExpression> DuckDBPyExpression::FunctionExpression(const string &function_name,

tools/pythonpkg/src/pyresult.cpp

@@ -177,7 +177,7 @@ unique_ptr<NumpyResultConversion> DuckDBPyResult::InitializeNumpyConversion(bool
 
 	auto conversion =
 	    make_uniq<NumpyResultConversion>(result->types, initial_capacity, result->client_properties, pandas);
-	return std::move(conversion);
+	return conversion;
 }
 
 py::dict DuckDBPyResult::FetchNumpyInternal(bool stream, idx_t vectors_per_chunk,