From aa3d0ad6504db517436fb559629da448527a8fd2 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Wed, 8 Oct 2025 14:21:17 +0200 Subject: [PATCH 01/14] 3.14 support --- .github/workflows/cleanup_pypi.yml | 4 +-- .github/workflows/code_quality.yml | 4 +-- .github/workflows/coverage.yml | 4 +-- .github/workflows/packaging_sdist.yml | 4 +-- .github/workflows/packaging_wheels.yml | 6 ++-- pyproject.toml | 10 +++---- tests/conftest.py | 41 ++++++++++++++++++++++++++ 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml index c4300be3..34fe3b53 100644 --- a/.github/workflows/cleanup_pypi.yml +++ b/.github/workflows/cleanup_pypi.yml @@ -50,9 +50,9 @@ jobs: exit 1 - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" - name: Run Cleanup env: diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 719e54ca..99b7884c 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -29,9 +29,9 @@ jobs: persist-credentials: false - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.9 - name: pre-commit (cache) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 41880382..6dd3db4a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -68,9 +68,9 @@ jobs: sudo apt-get -y install ccache - name: Install Astral UV and enable the cache - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.9 enable-cache: true cache-suffix: -${{ github.workflow }} diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 2723b437..249f161d 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -56,9 +56,9 @@ jobs: run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.11 - name: Build sdist diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b1a393a1..24aa0a74 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313 ] + python: [ cp39, cp310, cp311, cp312, cp313, cp314 ] platform: - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } @@ -78,9 +78,9 @@ jobs: run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" enable-cache: false cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} diff --git a/pyproject.toml b/pyproject.toml index 804c1ade..96c47675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas", # used for pandas dataframes all over the place - "pyarrow", # used for pyarrow support + "pyarrow; python_version < '3.14'", # used for pyarrow support "adbc-driver-manager", # for the adbc driver ] @@ -226,7 +226,7 @@ stubdeps = [ # dependencies used for typehints in the stubs "fsspec", "pandas", "polars", - "pyarrow", + "pyarrow; python_version < '3.14'", ] test = [ # dependencies used for running tests "adbc-driver-manager", @@ -248,8 +248,8 @@ test = [ # dependencies used for running tests "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0", - "pyarrow>=18.0.0", - "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", + "pyarrow>=18.0.0; python_version < '3.14'", + "torch>=2.2.2; python_version < '3.14' and ( sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13' )", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'", @@ -265,7 +265,7 @@ scripts = [ # dependencies used for running scripts "pandas", "pcpp", "polars", - "pyarrow", + "pyarrow; python_version < '3.14'", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script diff --git a/tests/conftest.py b/tests/conftest.py index bfb458a5..0f9985aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +import sys import warnings from importlib import import_module from pathlib import Path @@ -35,6 +36,46 @@ def import_pandas(): pytest.skip("Couldn't import pandas") +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item): + """Convert missing pyarrow imports to skips. + + TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. + https://github.com/duckdblabs/duckdb-internal/issues/6182 + """ + outcome = yield + if sys.version_info[:2] == (3, 14): + try: + outcome.get_result() + except ImportError as e: + if e.name == "pyarrow": + pytest.skip(f"pyarrow not available - {item.name} requires pyarrow") + else: + raise + + +@pytest.hookimpl(hookwrapper=True) +def pytest_make_collect_report(collector): + """Wrap module collection to catch pyarrow import errors on Python 3.14. + + If we're on Python 3.14 and a test module raises ModuleNotFoundError + for 'pyarrow', mark the entire module as xfailed rather than failing collection. + + TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. + https://github.com/duckdblabs/duckdb-internal/issues/6182 + """ + outcome = yield + result = outcome.get_result() + + if sys.version_info[:2] == (3, 14): + # Only handle failures from module collectors + if result.failed and collector.__class__.__name__ == "Module": + longrepr = str(result.longrepr) + if "ModuleNotFoundError: No module named 'pyarrow'" in longrepr: + result.outcome = "skipped" + result.longrepr = f"XFAIL: pyarrow not available {collector.name} ({longrepr.strip()})" + + # https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option # https://stackoverflow.com/a/47700320 def pytest_addoption(parser): From dbef096c27dce0d4a01d88f090d97ad5a3797723 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Wed, 8 Oct 2025 14:31:05 +0200 Subject: [PATCH 02/14] cibuildwheel bump --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 24aa0a74..bda10b0d 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -85,7 +85,7 @@ jobs: cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.0 + uses: pypa/cibuildwheel@v3.2 env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} From 18f5674e527beef31bd5b42eb71a4edad2ff0309 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Wed, 8 Oct 2025 17:28:39 +0200 Subject: [PATCH 03/14] fix pandas tests --- pyproject.toml | 3 +++ src/duckdb_py/pyresult.cpp | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 96c47675..ab6ffec4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,6 +233,9 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + "pytest-xdist", + "pytest-randomly", + "pytest-timestamper", "mypy", "coverage", "gcovr", diff --git a/src/duckdb_py/pyresult.cpp b/src/duckdb_py/pyresult.cpp index a2607a12..43edf0e1 100644 --- a/src/duckdb_py/pyresult.cpp +++ b/src/duckdb_py/pyresult.cpp @@ -23,6 +23,8 @@ #include "duckdb/main/chunk_scan_state/query_result.hpp" #include "duckdb/common/arrow/arrow_query_result.hpp" +using namespace pybind11::literals; + namespace duckdb { DuckDBPyResult::DuckDBPyResult(unique_ptr result_p) : result(std::move(result_p)) { @@ -293,8 +295,10 @@ void DuckDBPyResult::ChangeToTZType(PandasDataFrame &df) { if (result->types[i] == LogicalType::TIMESTAMP_TZ) { // first localize to UTC then convert to timezone_config auto utc_local = df[names[i].c_str()].attr("dt").attr("tz_localize")("UTC"); - df.attr("__setitem__")(names[i].c_str(), - utc_local.attr("dt").attr("tz_convert")(result->client_properties.time_zone)); + auto new_value = utc_local.attr("dt").attr("tz_convert")(result->client_properties.time_zone); + // We need to create the column anew because the exact dt changed to a new timezone + df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); + df.attr("__setitem__")(names[i].c_str(), new_value); } } } @@ -378,7 +382,9 @@ PandasDataFrame DuckDBPyResult::FrameFromNumpy(bool date_as_object, const py::ha if (date_as_object) { for (idx_t i = 0; i < result->ColumnCount(); i++) { if (result->types[i] == LogicalType::DATE) { - df.attr("__setitem__")(names[i].c_str(), df[names[i].c_str()].attr("dt").attr("date")); + auto new_value = df[names[i].c_str()].attr("dt").attr("date"); + df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); + df.attr("__setitem__")(names[i].c_str(), new_value); } } } From a7c0dd64f464a4571d953e439435b97cc5ba12d7 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 09:22:16 +0200 Subject: [PATCH 04/14] disable randomly --- .github/workflows/coverage.yml | 4 ++-- .github/workflows/packaging_sdist.yml | 2 +- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6dd3db4a..d019ab65 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -79,9 +79,9 @@ jobs: shell: bash run: | if [[ "${{ inputs.testsuite }}" == "all" ]]; then - uv run coverage run -m pytest ./tests --ignore=./tests/stubs + uv run coverage run -m pytest -n 4 ./tests elif [[ "${{ inputs.testsuite }}" == "fast" ]]; then - uv run coverage run -m pytest ./tests/fast + uv run coverage run -m pytest -n 4 ./tests/fast else echo "Invalid testsuite!" exit 1 diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 249f161d..3fee11ee 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -80,7 +80,7 @@ jobs: # run tests tests_root="${{ github.workspace }}/tests" tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}" - uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs + uv run --verbose pytest -n 4 $tests_dir --verbose - id: versioning run: | diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index bda10b0d..4855d703 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -50,10 +50,10 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && + uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest -n 4 ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose steps: - name: Checkout DuckDB Python diff --git a/pyproject.toml b/pyproject.toml index ab6ffec4..959c4ae2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,7 +234,7 @@ test = [ # dependencies used for running tests "pytest-reraise", "pytest-timeout", "pytest-xdist", - "pytest-randomly", + #"pytest-randomly", "pytest-timestamper", "mypy", "coverage", From e9ca6ebc3c005e252485929d0dd6e257b4eb6a10 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 12:04:46 +0200 Subject: [PATCH 05/14] fix tests --- .github/workflows/coverage.yml | 4 ++-- .github/workflows/packaging_sdist.yml | 2 +- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 2 +- tests/fast/api/test_duckdb_connection.py | 1 - .../arrow/test_arrow_fetch_recordbatch.py | 2 +- tests/fast/arrow/test_buffer_size_option.py | 8 +++---- tests/fast/arrow/test_timestamp_timezone.py | 2 +- tests/fast/test_expression.py | 4 ++-- tests/fast/test_parquet.py | 5 ++-- tests/fast/test_relation.py | 2 +- tests/fast/test_replacement_scan.py | 4 ++-- tests/fast/test_type.py | 23 +++++++++---------- tests/fast/test_type_explicit.py | 3 ++- tests/fast/test_value.py | 2 +- tests/fast/udf/test_null_filtering.py | 5 ++-- tests/fast/udf/test_remove_function.py | 2 +- tests/fast/udf/test_scalar.py | 2 +- tests/fast/udf/test_scalar_arrow.py | 2 +- tests/fast/udf/test_scalar_native.py | 2 +- tests/pytest.ini | 11 --------- 21 files changed, 40 insertions(+), 50 deletions(-) delete mode 100644 tests/pytest.ini diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d019ab65..fd62b6c1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -79,9 +79,9 @@ jobs: shell: bash run: | if [[ "${{ inputs.testsuite }}" == "all" ]]; then - uv run coverage run -m pytest -n 4 ./tests + uv run coverage run -m pytest ./tests elif [[ "${{ inputs.testsuite }}" == "fast" ]]; then - uv run coverage run -m pytest -n 4 ./tests/fast + uv run coverage run -m pytest ./tests/fast else echo "Invalid testsuite!" exit 1 diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 3fee11ee..de014538 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -80,7 +80,7 @@ jobs: # run tests tests_root="${{ github.workspace }}/tests" tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}" - uv run --verbose pytest -n 4 $tests_dir --verbose + uv run --verbose pytest $tests_dir --verbose - id: versioning run: | diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4855d703..bfb77fa2 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest -n 4 ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose steps: - name: Checkout DuckDB Python diff --git a/pyproject.toml b/pyproject.toml index 959c4ae2..ff6c6a77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,7 +305,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -q" +addopts = "-ra --numprocesses 4 --dist loadfile" testpaths = ["tests"] filterwarnings = [ "error", diff --git a/tests/fast/api/test_duckdb_connection.py b/tests/fast/api/test_duckdb_connection.py index d197e639..4fa32749 100644 --- a/tests/fast/api/test_duckdb_connection.py +++ b/tests/fast/api/test_duckdb_connection.py @@ -4,7 +4,6 @@ from conftest import ArrowPandas, NumpyPandas import duckdb -import duckdb.typing pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/arrow/test_arrow_fetch_recordbatch.py b/tests/fast/arrow/test_arrow_fetch_recordbatch.py index 0070430b..a5804c87 100644 --- a/tests/fast/arrow/test_arrow_fetch_recordbatch.py +++ b/tests/fast/arrow/test_arrow_fetch_recordbatch.py @@ -234,7 +234,7 @@ def test_record_batch_reader_from_relation(self, duckdb_cursor): duckdb_cursor = duckdb.connect() duckdb_cursor.execute("CREATE table t as select range a from range(3000);") relation = duckdb_cursor.table("t") - record_batch_reader = relation.record_batch() + record_batch_reader = relation.fetch_record_batch() chunk = record_batch_reader.read_next_batch() assert len(chunk) == 3000 diff --git a/tests/fast/arrow/test_buffer_size_option.py b/tests/fast/arrow/test_buffer_size_option.py index 28e3bf58..c63adfcc 100644 --- a/tests/fast/arrow/test_buffer_size_option.py +++ b/tests/fast/arrow/test_buffer_size_option.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import VARCHAR +from duckdb.sqltypes import VARCHAR pa = pytest.importorskip("pyarrow") @@ -13,21 +13,21 @@ def test_arrow_buffer_size(self): # All small string res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.string() # All Large String con.execute("SET arrow_large_buffer_size=True") res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.large_string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.large_string() # All small string again con.execute("SET arrow_large_buffer_size=False") res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.string() def test_arrow_buffer_size_udf(self): diff --git a/tests/fast/arrow/test_timestamp_timezone.py b/tests/fast/arrow/test_timestamp_timezone.py index 7e338626..27ddf3ac 100644 --- a/tests/fast/arrow/test_timestamp_timezone.py +++ b/tests/fast/arrow/test_timestamp_timezone.py @@ -64,7 +64,7 @@ def test_timestamp_stream(self, duckdb_cursor): con.execute("create table t (i timestamptz)") con.execute("insert into t values (NULL),('2021-11-15 02:30:00'::timestamptz)") rel = con.table("t") - arrow_tbl = rel.record_batch().read_all() + arrow_tbl = rel.fetch_record_batch().read_all() con.register("t2", arrow_tbl) assert con.execute("select * from t").fetchall() == con.execute("select * from t2").fetchall() diff --git a/tests/fast/test_expression.py b/tests/fast/test_expression.py index 5e61b455..c7eee6c1 100644 --- a/tests/fast/test_expression.py +++ b/tests/fast/test_expression.py @@ -13,7 +13,7 @@ LambdaExpression, StarExpression, ) -from duckdb.typing import INTEGER, TIMESTAMP, VARCHAR +from duckdb.sqltypes import INTEGER, TIMESTAMP, TINYINT, VARCHAR from duckdb.value.constant import IntegerValue, Value pytestmark = pytest.mark.skipif( @@ -804,7 +804,7 @@ def test_numeric_overflow(self): with pytest.raises(duckdb.OutOfRangeException, match="Overflow in multiplication of INT16"): rel2.fetchall() - val = duckdb.Value(100, duckdb.typing.TINYINT) + val = duckdb.Value(100, TINYINT) expr2 = ColumnExpression("salary") * val rel3 = rel.select(expr2) with pytest.raises(duckdb.OutOfRangeException, match="Overflow in multiplication of INT16"): diff --git a/tests/fast/test_parquet.py b/tests/fast/test_parquet.py index 3f6b1889..d418e1b9 100644 --- a/tests/fast/test_parquet.py +++ b/tests/fast/test_parquet.py @@ -3,9 +3,10 @@ import pytest import duckdb +import duckdb.sqltypes as duckdb_types -VARCHAR = duckdb.typing.VARCHAR -BIGINT = duckdb.typing.BIGINT +VARCHAR = duckdb_types.VARCHAR +BIGINT = duckdb_types.BIGINT filename = str(Path(__file__).parent / "data" / "binary_string.parquet") diff --git a/tests/fast/test_relation.py b/tests/fast/test_relation.py index f1b4b1fd..7b60a105 100644 --- a/tests/fast/test_relation.py +++ b/tests/fast/test_relation.py @@ -12,7 +12,7 @@ import duckdb from duckdb import ColumnExpression -from duckdb.typing import BIGINT, BOOLEAN, TINYINT, VARCHAR +from duckdb.sqltypes import BIGINT, BOOLEAN, TINYINT, VARCHAR @pytest.fixture(scope="session") diff --git a/tests/fast/test_replacement_scan.py b/tests/fast/test_replacement_scan.py index f0270167..57143b96 100644 --- a/tests/fast/test_replacement_scan.py +++ b/tests/fast/test_replacement_scan.py @@ -41,11 +41,11 @@ def fetch_arrow_table(rel): return rel.fetch_arrow_table() -def fetch_arrow_record_batch(rel): +def fetch_arrow_record_batch(rel: duckdb.DuckDBPyRelation): # Note: this has to executed first, otherwise we'll create a deadlock # Because it will try to execute the input at the same time as executing the relation # On the same connection (that's the core of the issue) - return rel.execute().record_batch() + return rel.execute().fetch_record_batch() def fetch_relation(rel): diff --git a/tests/fast/test_type.py b/tests/fast/test_type.py index 0eb96716..d8145166 100644 --- a/tests/fast/test_type.py +++ b/tests/fast/test_type.py @@ -4,8 +4,7 @@ import pytest import duckdb -import duckdb.typing -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BIT, BLOB, @@ -91,7 +90,7 @@ def test_incomplete_struct_type(self): with pytest.raises( duckdb.InvalidInputException, match="Could not convert empty dictionary to a duckdb STRUCT type" ): - duckdb.typing.DuckDBPyType({}) + DuckDBPyType({}) def test_map_type(self): type = duckdb.map_type(duckdb.sqltype("BIGINT"), duckdb.sqltype("DECIMAL(10, 2)")) @@ -228,26 +227,26 @@ def test_hash_method(self): # NOTE: we can support this, but I don't think going through hoops for an outdated version of python is worth it @pytest.mark.skipif(sys.version_info < (3, 9), reason="python3.7 does not store Optional[..] in a recognized way") def test_optional(self): - type = duckdb.typing.DuckDBPyType(Optional[str]) + type = DuckDBPyType(Optional[str]) assert type == "VARCHAR" - type = duckdb.typing.DuckDBPyType(Optional[Union[int, bool]]) + type = DuckDBPyType(Optional[Union[int, bool]]) assert type == "UNION(u1 BIGINT, u2 BOOLEAN)" - type = duckdb.typing.DuckDBPyType(Optional[list[int]]) + type = DuckDBPyType(Optional[list[int]]) assert type == "BIGINT[]" - type = duckdb.typing.DuckDBPyType(Optional[dict[int, str]]) + type = DuckDBPyType(Optional[dict[int, str]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) + type = DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) + type = DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[Union[Optional[str], Optional[bool]]]) + type = DuckDBPyType(Optional[Union[Optional[str], Optional[bool]]]) assert type == "UNION(u1 VARCHAR, u2 BOOLEAN)" - type = duckdb.typing.DuckDBPyType(Union[str, None]) + type = DuckDBPyType(Union[str, None]) assert type == "VARCHAR" @pytest.mark.skipif(sys.version_info < (3, 10), reason="'str | None' syntax requires Python 3.10 or higher") def test_optional_310(self): - type = duckdb.typing.DuckDBPyType(str | None) + type = DuckDBPyType(str | None) assert type == "VARCHAR" def test_children_attribute(self): diff --git a/tests/fast/test_type_explicit.py b/tests/fast/test_type_explicit.py index 3b9fe334..f8a5ca63 100644 --- a/tests/fast/test_type_explicit.py +++ b/tests/fast/test_type_explicit.py @@ -1,4 +1,5 @@ import duckdb +import duckdb.sqltypes as duckdb_types class TestMap: @@ -9,7 +10,7 @@ def test_array_list_tuple_ambiguity(self): # By using an explicit duckdb.Value with an array type, we should convert the input as an array # and get an array (tuple) back - typ = duckdb.array_type(duckdb.typing.BIGINT, 2) + typ = duckdb.array_type(duckdb_types.BIGINT, 2) val = duckdb.Value((1, 2), typ) res = con.sql("SELECT $arg", params={"arg": val}).fetchall()[0][0] assert res == (1, 2) diff --git a/tests/fast/test_value.py b/tests/fast/test_value.py index 58aa7a4d..bf66bee8 100644 --- a/tests/fast/test_value.py +++ b/tests/fast/test_value.py @@ -6,7 +6,7 @@ import duckdb from duckdb import InvalidInputException, NotImplementedException -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BIT, BLOB, diff --git a/tests/fast/udf/test_null_filtering.py b/tests/fast/udf/test_null_filtering.py index e5c0d546..8bf2ce73 100644 --- a/tests/fast/udf/test_null_filtering.py +++ b/tests/fast/udf/test_null_filtering.py @@ -5,7 +5,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BLOB, BOOLEAN, @@ -24,6 +24,7 @@ UTINYINT, UUID, VARCHAR, + DuckDBPyType, ) pd = pytest.importorskip("pandas") @@ -31,7 +32,7 @@ class Candidate(NamedTuple): - type: duckdb.typing.DuckDBPyType + type: DuckDBPyType variant_one: Any variant_two: Any diff --git a/tests/fast/udf/test_remove_function.py b/tests/fast/udf/test_remove_function.py index 7ced339a..8ed739a4 100644 --- a/tests/fast/udf/test_remove_function.py +++ b/tests/fast/udf/test_remove_function.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import BIGINT, VARCHAR +from duckdb.sqltypes import BIGINT, VARCHAR pd = pytest.importorskip("pandas") pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/udf/test_scalar.py b/tests/fast/udf/test_scalar.py index 40e0d4de..80594c98 100644 --- a/tests/fast/udf/test_scalar.py +++ b/tests/fast/udf/test_scalar.py @@ -7,7 +7,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BLOB, BOOLEAN, diff --git a/tests/fast/udf/test_scalar_arrow.py b/tests/fast/udf/test_scalar_arrow.py index 46f932a1..e3f18344 100644 --- a/tests/fast/udf/test_scalar_arrow.py +++ b/tests/fast/udf/test_scalar_arrow.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import BIGINT, INTEGER, VARCHAR +from duckdb.sqltypes import BIGINT, INTEGER, VARCHAR pd = pytest.importorskip("pandas") pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/udf/test_scalar_native.py b/tests/fast/udf/test_scalar_native.py index 64ea5b5b..3c6b3cc8 100644 --- a/tests/fast/udf/test_scalar_native.py +++ b/tests/fast/udf/test_scalar_native.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, HUGEINT, INTEGER, diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index 0c17afd5..00000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -# pytest.ini -[pytest] -filterwarnings = - error - ignore::UserWarning - ignore::DeprecationWarning - # Jupyter is throwing DeprecationWarnings - ignore:function ham\(\) is deprecated:DeprecationWarning - # Pyspark is throwing these warnings - ignore:distutils Version classes are deprecated:DeprecationWarning - ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning From 95dc95948d84abbc0897a184d771c0c4f6c82fea Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 13:07:30 +0200 Subject: [PATCH 06/14] make sure config is loaded --- .github/workflows/packaging_sdist.yml | 2 +- .github/workflows/packaging_wheels.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index de014538..b6558744 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -80,7 +80,7 @@ jobs: # run tests tests_root="${{ github.workspace }}/tests" tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}" - uv run --verbose pytest $tests_dir --verbose + uv run --verbose pytest -c ${{ github.workspace }}/pyproject.toml $tests_dir - id: versioning run: | diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index bfb77fa2..1612620f 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose + uv run -v pytest -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python diff --git a/pyproject.toml b/pyproject.toml index ff6c6a77..c975ca99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,7 +305,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra --numprocesses 4 --dist loadfile" +addopts = "-ra --numprocesses 4 --dist loadfile --verbose" testpaths = ["tests"] filterwarnings = [ "error", From 618e5b3424de5d5a60faeb981ecf0e31aae7bc9e Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 15:43:50 +0200 Subject: [PATCH 07/14] replace use of deprecated typing._UnionGenericAlias --- pyproject.toml | 2 +- scripts/cache_data.json | 15 ++++++++++++++- scripts/generate_import_cache_cpp.py | 10 +++++----- scripts/imports.py | 3 ++- .../import_cache/modules/collections_module.hpp | 8 ++++---- .../import_cache/modules/datetime_module.hpp | 8 ++++---- .../import_cache/modules/decimal_module.hpp | 8 ++++---- .../import_cache/modules/duckdb_module.hpp | 8 ++++---- .../import_cache/modules/ipython_module.hpp | 8 ++++---- .../import_cache/modules/ipywidgets_module.hpp | 8 ++++---- .../import_cache/modules/numpy_module.hpp | 8 ++++---- .../import_cache/modules/pandas_module.hpp | 8 ++++---- .../import_cache/modules/pathlib_module.hpp | 8 ++++---- .../import_cache/modules/polars_module.hpp | 8 ++++---- .../import_cache/modules/pyarrow_module.hpp | 8 ++++---- .../import_cache/modules/pytz_module.hpp | 8 ++++---- .../import_cache/modules/types_module.hpp | 8 ++++---- .../import_cache/modules/typing_module.hpp | 13 +++++++------ .../import_cache/modules/uuid_module.hpp | 8 ++++---- .../import_cache/python_import_cache_modules.hpp | 2 +- src/duckdb_py/typing/pytype.cpp | 13 ++++++------- 21 files changed, 92 insertions(+), 78 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c975ca99..9c31b329 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,7 +305,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra --numprocesses 4 --dist loadfile --verbose" +addopts = "-ra -p no:xdist --verbose" testpaths = ["tests"] filterwarnings = [ "error", diff --git a/scripts/cache_data.json b/scripts/cache_data.json index 3dd9a1f1..b1f1df3e 100644 --- a/scripts/cache_data.json +++ b/scripts/cache_data.json @@ -622,7 +622,8 @@ "full_path": "typing", "name": "typing", "children": [ - "typing._UnionGenericAlias" + "typing.Union", + "typing.get_origin" ] }, "typing._UnionGenericAlias": { @@ -793,5 +794,17 @@ "full_path": "pyarrow.decimal128", "name": "decimal128", "children": [] + }, + "typing.get_origin": { + "type": "attribute", + "full_path": "typing.get_origin", + "name": "get_origin", + "children": [] + }, + "typing.Union": { + "type": "attribute", + "full_path": "typing.Union", + "name": "Union", + "children": [] } } \ No newline at end of file diff --git a/scripts/generate_import_cache_cpp.py b/scripts/generate_import_cache_cpp.py index d10dde0c..04efffde 100644 --- a/scripts/generate_import_cache_cpp.py +++ b/scripts/generate_import_cache_cpp.py @@ -151,10 +151,10 @@ def to_string(self): //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb {{ {self.get_classes()} @@ -230,7 +230,7 @@ def get_root_modules(files: list[ModuleFile]): def get_module_file_path_includes(files: list[ModuleFile]): - template = '#include "duckdb_python/import_cache/modules/{}' + template = '#include "duckdb_python/import_cache/modules/{}"' return "\n".join(template.format(f.file_name) for f in files) diff --git a/scripts/imports.py b/scripts/imports.py index d7e38750..ee4c4597 100644 --- a/scripts/imports.py +++ b/scripts/imports.py @@ -128,7 +128,8 @@ import typing -typing._UnionGenericAlias +typing.Union +typing.get_origin import uuid diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp index 08bce28d..d8e3ad03 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp index 508d5133..c05f77d4 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp index 15f4ecf6..7c979329 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp index 2272a7d7..29a6dc3f 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp index 2742e19b..91a16fb6 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp index 26916b8d..0b9a10f1 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp index 041e854b..e67ae893 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp index ef0de3b4..05ff25e2 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp index 1ef2d473..1025f487 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp index 04db331b..c22173b4 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp index d3331565..ff9d9ebc 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp index 1aca4e2e..ccdde0c8 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp index f88b8a32..9de7dc3a 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp index 7ebf3696..38e29331 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { @@ -26,12 +26,13 @@ struct TypingCacheItem : public PythonImportCacheItem { static constexpr const char *Name = "typing"; public: - TypingCacheItem() : PythonImportCacheItem("typing"), _UnionGenericAlias("_UnionGenericAlias", this) { + TypingCacheItem() : PythonImportCacheItem("typing"), Union("Union", this), get_origin("get_origin", this) { } ~TypingCacheItem() override { } - PythonImportCacheItem _UnionGenericAlias; + PythonImportCacheItem Union; + PythonImportCacheItem get_origin; }; } // namespace duckdb diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp index 7bd3a6c2..f3108848 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp b/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp index 172c0513..e82cd2dc 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp @@ -12,4 +12,4 @@ #include "duckdb_python/import_cache/modules/types_module.hpp" #include "duckdb_python/import_cache/modules/typing_module.hpp" #include "duckdb_python/import_cache/modules/uuid_module.hpp" -#include "duckdb_python/import_cache/modules/collections_module.hpp" +#include "duckdb_python/import_cache/modules/collections_module.hpp" \ No newline at end of file diff --git a/src/duckdb_py/typing/pytype.cpp b/src/duckdb_py/typing/pytype.cpp index eca92bed..9f049b99 100644 --- a/src/duckdb_py/typing/pytype.cpp +++ b/src/duckdb_py/typing/pytype.cpp @@ -20,17 +20,16 @@ bool PyGenericAlias::check_(const py::handle &object) { // NOLINTNEXTLINE(readability-identifier-naming) bool PyUnionType::check_(const py::handle &object) { auto types_loaded = ModuleIsLoaded(); - auto typing_loaded = ModuleIsLoaded(); - - if (!types_loaded && !typing_loaded) { - return false; - } - auto &import_cache = *DuckDBPyConnection::ImportCache(); + + // for >= py310: isinstance(object, types.UnionType) if (types_loaded && py::isinstance(object, import_cache.types.UnionType())) { return true; } - if (typing_loaded && py::isinstance(object, import_cache.typing._UnionGenericAlias())) { + // for all py3: isinstance(typing.get_origin(object), typing.Union) + auto get_origin_func = import_cache.typing.get_origin(); + auto origin = get_origin_func(object); + if (py::isinstance(origin, import_cache.typing.Union())) { return true; } return false; From 97f2255d0e536db931983372361e1b7d53f98a9f Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 16:09:47 +0200 Subject: [PATCH 08/14] equality instead of instanceof --- src/duckdb_py/typing/pytype.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/duckdb_py/typing/pytype.cpp b/src/duckdb_py/typing/pytype.cpp index 9f049b99..e7e31a18 100644 --- a/src/duckdb_py/typing/pytype.cpp +++ b/src/duckdb_py/typing/pytype.cpp @@ -26,10 +26,10 @@ bool PyUnionType::check_(const py::handle &object) { if (types_loaded && py::isinstance(object, import_cache.types.UnionType())) { return true; } - // for all py3: isinstance(typing.get_origin(object), typing.Union) + // for all py3: typing.get_origin(object) is typing.Union auto get_origin_func = import_cache.typing.get_origin(); auto origin = get_origin_func(object); - if (py::isinstance(origin, import_cache.typing.Union())) { + if (origin.is(import_cache.typing.Union())) { return true; } return false; From 71a8e9aae204afae4fdcc2243f5d2d114f9dbca2 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 16:58:37 +0200 Subject: [PATCH 09/14] fix pytest collect hook --- tests/conftest.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0f9985aa..f2ad7cec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,15 +65,16 @@ def pytest_make_collect_report(collector): https://github.com/duckdblabs/duckdb-internal/issues/6182 """ outcome = yield - result = outcome.get_result() + report: pytest.CollectReport = outcome.get_result() if sys.version_info[:2] == (3, 14): # Only handle failures from module collectors - if result.failed and collector.__class__.__name__ == "Module": - longrepr = str(result.longrepr) - if "ModuleNotFoundError: No module named 'pyarrow'" in longrepr: - result.outcome = "skipped" - result.longrepr = f"XFAIL: pyarrow not available {collector.name} ({longrepr.strip()})" + if report.failed and collector.__class__.__name__ == "Module": + longreprtext = report.longreprtext + if "ModuleNotFoundError: No module named 'pyarrow'" in longreprtext: + report.outcome = "skipped" + reason = f"XFAIL: [pyarrow not available] {longreprtext}" + report.longrepr = (report.fspath, None, reason) # https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option From 230bada7413ac35219a4a43bcbf9a6d3e4e24a5e Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 17:11:35 +0200 Subject: [PATCH 10/14] try to get pytest config work correct for windows runners --- .github/workflows/packaging_wheels.yml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 1612620f..4a10e136 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -48,12 +48,12 @@ jobs: runs-on: ${{ matrix.platform.os }} env: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} - CIBW_TEST_SOURCES: tests + CIBW_TEST_SOURCES: tests pyproject.toml CIBW_BEFORE_TEST: > uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python diff --git a/pyproject.toml b/pyproject.toml index 9c31b329..c975ca99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,7 +305,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -p no:xdist --verbose" +addopts = "-ra --numprocesses 4 --dist loadfile --verbose" testpaths = ["tests"] filterwarnings = [ "error", From ae6e43c6b30bf38d2b5e2477a450f29c58ac0e5c Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 18:01:12 +0200 Subject: [PATCH 11/14] another try to appease the windows runners --- .github/workflows/packaging_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4a10e136..e4b12641 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -48,12 +48,12 @@ jobs: runs-on: ${{ matrix.platform.os }} env: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} - CIBW_TEST_SOURCES: tests pyproject.toml + CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} + uv run -v pytest --rootdir $PWD -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python From cade3446736d09e645bb871b2a9eeac0e5d206bb Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 20:09:43 +0200 Subject: [PATCH 12/14] without PWD --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index e4b12641..7359f91c 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest --rootdir $PWD -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} + uv run -v pytest --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python From 8058a017c6aad0c144d241535e907eef7dc837c6 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 9 Oct 2025 21:18:37 +0200 Subject: [PATCH 13/14] trying confcutdir --- .github/workflows/packaging_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 7359f91c..002b3b6a 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -53,7 +53,7 @@ jobs: uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} + uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python From 3ee4f0299eb122f67f9ab448dcfdb6f56a8405fd Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Fri, 10 Oct 2025 10:08:12 +0200 Subject: [PATCH 14/14] remove pytest plugins --- .github/workflows/packaging_wheels.yml | 13 +++++++++++++ cmake/compiler_launcher.cmake | 6 ++---- pyproject.toml | 4 +--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 002b3b6a..a2e4f857 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -44,9 +44,22 @@ jobs: - { minimal: true, python: cp310 } - { minimal: true, python: cp311 } - { minimal: true, python: cp312 } + - { minimal: true, python: cp313 } - { minimal: true, platform: { arch: universal2 } } runs-on: ${{ matrix.platform.os }} env: + ### cibuildwheel configuration + # + # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others): + # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it + # built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make + # sure that the pytest config in pyproject.toml is available. + # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing, + # `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments + # across all platforms we build for. This might be solved in newer uv versions. + # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from + # traversing the full filesystem, which produces an error on Windows. + # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64. CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > diff --git a/cmake/compiler_launcher.cmake b/cmake/compiler_launcher.cmake index 8f77da86..faa4b45c 100644 --- a/cmake/compiler_launcher.cmake +++ b/cmake/compiler_launcher.cmake @@ -8,8 +8,7 @@ include(CMakeParseArguments) # Function to look for ccache and sccache to speed up builds, if available # ──────────────────────────────────────────── function(setup_compiler_launcher_if_available) - if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED - ENV{CMAKE_C_COMPILER_LAUNCHER}) + if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER) find_program(COMPILER_LAUNCHER NAMES ccache sccache) if(COMPILER_LAUNCHER) message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher") @@ -19,8 +18,7 @@ function(setup_compiler_launcher_if_available) endif() endif() - if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER - AND NOT DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER}) + if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) find_program(COMPILER_LAUNCHER NAMES ccache sccache) if(COMPILER_LAUNCHER) message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher") diff --git a/pyproject.toml b/pyproject.toml index c975ca99..af75d9a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,8 +233,6 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", - "pytest-xdist", - #"pytest-randomly", "pytest-timestamper", "mypy", "coverage", @@ -305,7 +303,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra --numprocesses 4 --dist loadfile --verbose" +addopts = "-ra --verbose" testpaths = ["tests"] filterwarnings = [ "error",