Run the python test suite when building for Pyodide

.github/workflows/Pyodide.yml

@@ -44,8 +44,10 @@ jobs:
         version:
           - python: "3.10"
             pyodide-build: "0.22.1"
+            node: "16"
           - python: "3.11"
             pyodide-build: "0.25.1"
+            node: "18"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -76,22 +78,28 @@ jobs:
           CFLAGS: "-fexceptions"
           LDFLAGS: "-fexceptions"
 
-      - name: smoke test duckdb on pyodide
+      - name: install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.version.node }}
+
+      - name: create pyodide environment
+        run: pyodide venv .venv-pyodide
+
+      - name: install deps into environment
         run: |
-          pyodide venv .venv-pyodide
           source .venv-pyodide/bin/activate
-          pip install ./tools/pythonpkg/dist/*.whl
-
-          python -V
+          pip install pytest numpy pandas mypy
 
-          python <<EOF
-          import duckdb
-          print(duckdb.__version__)
-          print(duckdb.sql("SELECT 1 AS a"))
+      - name: install duckdb wasm wheel into environment
+        run: |
+          source .venv-pyodide/bin/activate
+          pip install ./tools/pythonpkg/dist/*.whl
 
-          (platform,) = duckdb.execute("PRAGMA platform").fetchone()
-          assert platform == "wasm_eh_pyodide", platform
-          EOF
+      - name: run fast tests pyodide
+        run: |
+          source .venv-pyodide/bin/activate
+          python -m pytest ./tools/pythonpkg/tests
 
       - name: Wheel sizes
         run: |

tools/pythonpkg/tests/extensions/test_extensions_loading.py

@@ -1,10 +1,17 @@
 import os
+import platform
 
 import duckdb
 from pytest import raises
 import pytest
 
 
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Extensions are not supported on Emscripten",
+)
+
+
 def test_extension_loading(require):
     if not os.getenv('DUCKDB_PYTHON_TEST_EXTENSION_REQUIRED', False):
         return

tools/pythonpkg/tests/fast/api/test_connection_interrupt.py

@@ -1,3 +1,4 @@
+import platform
 import threading
 import time
 
@@ -6,6 +7,10 @@
 
 
 class TestConnectionInterrupt(object):
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="threads not allowed on Emscripten",
+    )
     def test_connection_interrupt(self):
         conn = duckdb.connect()
 

tools/pythonpkg/tests/fast/api/test_query_interrupt.py

@@ -2,6 +2,7 @@
 import time
 import pytest
 
+import platform
 import threading
 import _thread as thread
 
@@ -14,6 +15,10 @@ def send_keyboard_interrupt():
 
 
 class TestQueryInterruption(object):
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="Emscripten builds cannot use threads",
+    )
     def test_query_interruption(self):
         con = duckdb.connect()
         thread = threading.Thread(target=send_keyboard_interrupt)

tools/pythonpkg/tests/fast/api/test_read_csv.py

@@ -1,11 +1,9 @@
 from multiprocessing.sharedctypes import Value
-import numpy
 import datetime
-import pandas
 import pytest
+import platform
 import duckdb
 from io import StringIO, BytesIO
-from duckdb.typing import BIGINT, VARCHAR, INTEGER
 
 
 def TestFile(name):
@@ -441,6 +439,7 @@ def test_read_csv_glob(self, tmp_path, create_temp_csv):
         res = con.sql("select * from rel order by all").fetchall()
         assert res == [(1,), (2,), (3,), (4,), (5,), (6,)]
 
+    @pytest.mark.xfail(condition=platform.system() == "Emscripten", reason="time zones not working")
     def test_read_csv_combined(self, duckdb_cursor):
         CSV_FILE = TestFile('stress_test.csv')
         COLUMNS = {
@@ -470,6 +469,8 @@ def test_read_csv_combined(self, duckdb_cursor):
         assert rel.types == rel2.types
 
     def test_read_csv_names(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         rel = con.read_csv(file, names=['a', 'b', 'c'])
@@ -488,6 +489,8 @@ def test_read_csv_names(self):
             assert rel.columns == ['a', 'b', 'a', 'b']
 
     def test_read_csv_names_mixed_with_dtypes(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         rel = con.read_csv(
@@ -518,6 +521,8 @@ def test_read_csv_names_mixed_with_dtypes(self):
             )
 
     def test_read_csv_multi_file(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file1 = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         file2 = StringIO('one,two,three,four\n5,6,7,8\n5,6,7,8\n5,6,7,8')
@@ -547,6 +552,7 @@ def test_read_csv_empty_list(self):
             res = rel.fetchall()
 
     def test_read_csv_list_invalid_path(self):
+        pytest.importorskip("fsspec")
         con = duckdb.connect()
         files = [
             StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4'),

tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py

@@ -1,6 +1,7 @@
 import duckdb
 import datetime
 import numpy as np
+import platform
 import pytest
 import decimal
 import math
@@ -528,6 +529,10 @@ def test_double_object_conversion(self, pandas, duckdb_cursor):
         assert isinstance(converted_col['0'].dtype, double_dtype.__class__) == True
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="older numpy raises a warning when running with Pyodide",
+    )
     def test_numpy_object_with_stride(self, pandas, duckdb_cursor):
         df = pandas.DataFrame(columns=["idx", "evens", "zeros"])
 

tools/pythonpkg/tests/fast/pandas/test_timedelta.py

@@ -1,10 +1,16 @@
+import platform
 import pandas as pd
 import duckdb
 import datetime
-import numpy as np
 import pytest
 
 
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Not supported on Emscripten",
+)
+
+
 class TestTimedelta(object):
     def test_timedelta_positive(self, duckdb_cursor):
         duckdb_interval = duckdb_cursor.query(

tools/pythonpkg/tests/fast/pandas/test_timestamp.py

@@ -1,8 +1,8 @@
 import duckdb
-import os
 import datetime
 import pytest
 import pandas as pd
+import platform
 from conftest import pandas_2_or_higher
 
 
@@ -64,6 +64,7 @@ def test_timestamp_timedelta(self):
         df_from_duck = duckdb.from_df(df).df()
         assert df_from_duck.equals(df)
 
+    @pytest.mark.xfail(condition=platform.system() == "Emscripten", reason="time zones not working")
     def test_timestamp_timezone(self, duckdb_cursor):
         rel = duckdb_cursor.query(
             """

tools/pythonpkg/tests/fast/relational_api/test_rapi_query.py

@@ -1,5 +1,11 @@
 import duckdb
 import pytest
+import platform
+
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Not supported on Emscripten",
+)
 
 
 @pytest.fixture()

tools/pythonpkg/tests/fast/test_alex_multithread.py

@@ -1,10 +1,15 @@
+import platform
 import duckdb
 from threading import Thread, current_thread
-import pandas as pd
-import os
 import pytest
 
 
+pytestmark = pytest.mark.xfail(
+    condition=platform.system() == "Emscripten",
+    reason="Emscripten builds cannot use threads",
+)
+
+
 @pytest.fixture(scope="session")
 def tmp_database(tmp_path_factory):
     database = tmp_path_factory.mktemp("databases", numbered=True) / "tmp.duckdb"

tools/pythonpkg/tests/fast/test_memory_leaks.py

@@ -1,9 +1,10 @@
 import gc
-import duckdb
 import pytest
-import os, psutil
+import os
 import pandas as pd
 
+psutil = pytest.importorskip("psutil")
+
 
 @pytest.fixture
 def check_leaks():

tools/pythonpkg/tests/fast/test_multithread.py

@@ -1,3 +1,4 @@
+import platform
 import duckdb
 import pytest
 import threading
@@ -7,12 +8,11 @@
 import os
 from typing import List
 
-try:
-    import pyarrow as pa
 
-    can_run = True
-except ImportError:
-    can_run = False
+pytestmark = pytest.mark.xfail(
+    condition=platform.system() == "Emscripten",
+    reason="Emscripten builds cannot use threads",
+)
 
 
 def connect_duck(duckdb_conn):
@@ -415,15 +415,13 @@ def test_fetchdfchunk(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_fetcharrow(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, fetch_arrow_query, pandas)
         duck_threads.multithread_test()
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_fetch_record_batch(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, fetch_record_batch_query, pandas)
         duck_threads.multithread_test()
 
@@ -449,8 +447,7 @@ def test_df_unregister(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_arrow_register_unregister(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, arrow_register_unregister, pandas)
         duck_threads.multithread_test()
 
@@ -481,8 +478,7 @@ def test_from_DF(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_from_arrow(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, from_arrow, pandas)
         duck_threads.multithread_test()
 

tools/pythonpkg/tests/fast/test_relation_dependency_leak.py

@@ -1,6 +1,5 @@
-import duckdb
 import numpy as np
-import os, psutil
+import os
 import pytest
 
 try:
@@ -12,6 +11,9 @@
 from conftest import NumpyPandas, ArrowPandas
 
 
+psutil = pytest.importorskip("psutil")
+
+
 def check_memory(function_to_check, pandas, duckdb_cursor):
     process = psutil.Process(os.getpid())
     mem_usage = process.memory_info().rss / (10**9)