From 9e81771ed0eaf674a9c0f89559f5eed2f3a8c4f4 Mon Sep 17 00:00:00 2001
From: Phillip Cloud <417981+cpcloud@users.noreply.github.com>
Date: Thu, 2 May 2024 14:03:49 -0400
Subject: [PATCH] Run the python test suite when building for Pyodide

---
 .github/workflows/Pyodide.yml                 | 32 ++++++++++++-------
 .../extensions/test_extensions_loading.py     |  7 ++++
 .../fast/api/test_connection_interrupt.py     |  5 +++
 .../tests/fast/api/test_query_interrupt.py    |  5 +++
 .../pythonpkg/tests/fast/api/test_read_csv.py | 12 +++++--
 .../fast/pandas/test_df_object_resolution.py  |  5 +++
 .../tests/fast/pandas/test_timedelta.py       |  8 ++++-
 .../tests/fast/pandas/test_timestamp.py       |  3 +-
 .../fast/relational_api/test_rapi_query.py    |  6 ++++
 .../tests/fast/test_alex_multithread.py       |  9 ++++--
 .../pythonpkg/tests/fast/test_memory_leaks.py |  5 +--
 .../pythonpkg/tests/fast/test_multithread.py  | 22 ++++++-------
 .../fast/test_relation_dependency_leak.py     |  6 ++--
 13 files changed, 89 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/Pyodide.yml b/.github/workflows/Pyodide.yml
index 160bcc759f56..5f4861459e33 100644
--- a/.github/workflows/Pyodide.yml
+++ b/.github/workflows/Pyodide.yml
@@ -44,8 +44,10 @@ jobs:
         version:
           - python: "3.10"
             pyodide-build: "0.22.1"
+            node: "16"
           - python: "3.11"
             pyodide-build: "0.25.1"
+            node: "18"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -76,22 +78,28 @@ jobs:
           CFLAGS: "-fexceptions"
           LDFLAGS: "-fexceptions"
 
-      - name: smoke test duckdb on pyodide
+      - name: install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.version.node }}
+
+      - name: create pyodide environment
+        run: pyodide venv .venv-pyodide
+
+      - name: install deps into environment
         run: |
-          pyodide venv .venv-pyodide
           source .venv-pyodide/bin/activate
-          pip install ./tools/pythonpkg/dist/*.whl
-
-          python -V
+          pip install pytest numpy pandas mypy
 
-          python <<EOF
-          import duckdb
-          print(duckdb.__version__)
-          print(duckdb.sql("SELECT 1 AS a"))
+      - name: install duckdb wasm wheel into environment
+        run: |
+          source .venv-pyodide/bin/activate
+          pip install ./tools/pythonpkg/dist/*.whl
 
-          (platform,) = duckdb.execute("PRAGMA platform").fetchone()
-          assert platform == "wasm_eh_pyodide", platform
-          EOF
+      - name: run fast tests pyodide
+        run: |
+          source .venv-pyodide/bin/activate
+          python -m pytest ./tools/pythonpkg/tests
 
       - name: Wheel sizes
         run: |
diff --git a/tools/pythonpkg/tests/extensions/test_extensions_loading.py b/tools/pythonpkg/tests/extensions/test_extensions_loading.py
index a3ecfe3a4e65..c045e4a330cc 100644
--- a/tools/pythonpkg/tests/extensions/test_extensions_loading.py
+++ b/tools/pythonpkg/tests/extensions/test_extensions_loading.py
@@ -1,10 +1,17 @@
 import os
+import platform
 
 import duckdb
 from pytest import raises
 import pytest
 
 
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Extensions are not supported on Emscripten",
+)
+
+
 def test_extension_loading(require):
     if not os.getenv('DUCKDB_PYTHON_TEST_EXTENSION_REQUIRED', False):
         return
diff --git a/tools/pythonpkg/tests/fast/api/test_connection_interrupt.py b/tools/pythonpkg/tests/fast/api/test_connection_interrupt.py
index 9eeb1f45ffcc..4efd68b59918 100644
--- a/tools/pythonpkg/tests/fast/api/test_connection_interrupt.py
+++ b/tools/pythonpkg/tests/fast/api/test_connection_interrupt.py
@@ -1,3 +1,4 @@
+import platform
 import threading
 import time
 
@@ -6,6 +7,10 @@
 
 
 class TestConnectionInterrupt(object):
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="threads not allowed on Emscripten",
+    )
     def test_connection_interrupt(self):
         conn = duckdb.connect()
 
diff --git a/tools/pythonpkg/tests/fast/api/test_query_interrupt.py b/tools/pythonpkg/tests/fast/api/test_query_interrupt.py
index 43017253f033..6334e475d27f 100644
--- a/tools/pythonpkg/tests/fast/api/test_query_interrupt.py
+++ b/tools/pythonpkg/tests/fast/api/test_query_interrupt.py
@@ -2,6 +2,7 @@
 import time
 import pytest
 
+import platform
 import threading
 import _thread as thread
 
@@ -14,6 +15,10 @@ def send_keyboard_interrupt():
 
 
 class TestQueryInterruption(object):
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="Emscripten builds cannot use threads",
+    )
     def test_query_interruption(self):
         con = duckdb.connect()
         thread = threading.Thread(target=send_keyboard_interrupt)
diff --git a/tools/pythonpkg/tests/fast/api/test_read_csv.py b/tools/pythonpkg/tests/fast/api/test_read_csv.py
index 49df1ac4f976..c5f414098be8 100644
--- a/tools/pythonpkg/tests/fast/api/test_read_csv.py
+++ b/tools/pythonpkg/tests/fast/api/test_read_csv.py
@@ -1,11 +1,9 @@
 from multiprocessing.sharedctypes import Value
-import numpy
 import datetime
-import pandas
 import pytest
+import platform
 import duckdb
 from io import StringIO, BytesIO
-from duckdb.typing import BIGINT, VARCHAR, INTEGER
 
 
 def TestFile(name):
@@ -441,6 +439,7 @@ def test_read_csv_glob(self, tmp_path, create_temp_csv):
         res = con.sql("select * from rel order by all").fetchall()
         assert res == [(1,), (2,), (3,), (4,), (5,), (6,)]
 
+    @pytest.mark.xfail(condition=platform.system() == "Emscripten", reason="time zones not working")
     def test_read_csv_combined(self, duckdb_cursor):
         CSV_FILE = TestFile('stress_test.csv')
         COLUMNS = {
@@ -470,6 +469,8 @@ def test_read_csv_combined(self, duckdb_cursor):
         assert rel.types == rel2.types
 
     def test_read_csv_names(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         rel = con.read_csv(file, names=['a', 'b', 'c'])
@@ -488,6 +489,8 @@ def test_read_csv_names(self):
             assert rel.columns == ['a', 'b', 'a', 'b']
 
     def test_read_csv_names_mixed_with_dtypes(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         rel = con.read_csv(
@@ -518,6 +521,8 @@ def test_read_csv_names_mixed_with_dtypes(self):
             )
 
     def test_read_csv_multi_file(self):
+        pytest.importorskip("fsspec")
+
         con = duckdb.connect()
         file1 = StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4')
         file2 = StringIO('one,two,three,four\n5,6,7,8\n5,6,7,8\n5,6,7,8')
@@ -547,6 +552,7 @@ def test_read_csv_empty_list(self):
             res = rel.fetchall()
 
     def test_read_csv_list_invalid_path(self):
+        pytest.importorskip("fsspec")
         con = duckdb.connect()
         files = [
             StringIO('one,two,three,four\n1,2,3,4\n1,2,3,4\n1,2,3,4'),
diff --git a/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py b/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py
index 1a07e47fc8f4..f2b5a98a352a 100644
--- a/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py
+++ b/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py
@@ -1,6 +1,7 @@
 import duckdb
 import datetime
 import numpy as np
+import platform
 import pytest
 import decimal
 import math
@@ -528,6 +529,10 @@ def test_double_object_conversion(self, pandas, duckdb_cursor):
         assert isinstance(converted_col['0'].dtype, double_dtype.__class__) == True
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
+    @pytest.mark.xfail(
+        condition=platform.system() == "Emscripten",
+        reason="older numpy raises a warning when running with Pyodide",
+    )
     def test_numpy_object_with_stride(self, pandas, duckdb_cursor):
         df = pandas.DataFrame(columns=["idx", "evens", "zeros"])
 
diff --git a/tools/pythonpkg/tests/fast/pandas/test_timedelta.py b/tools/pythonpkg/tests/fast/pandas/test_timedelta.py
index 843c47f2df64..1e10399f82a1 100644
--- a/tools/pythonpkg/tests/fast/pandas/test_timedelta.py
+++ b/tools/pythonpkg/tests/fast/pandas/test_timedelta.py
@@ -1,10 +1,16 @@
+import platform
 import pandas as pd
 import duckdb
 import datetime
-import numpy as np
 import pytest
 
 
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Not supported on Emscripten",
+)
+
+
 class TestTimedelta(object):
     def test_timedelta_positive(self, duckdb_cursor):
         duckdb_interval = duckdb_cursor.query(
diff --git a/tools/pythonpkg/tests/fast/pandas/test_timestamp.py b/tools/pythonpkg/tests/fast/pandas/test_timestamp.py
index 51641e281a34..5e72acd85e55 100644
--- a/tools/pythonpkg/tests/fast/pandas/test_timestamp.py
+++ b/tools/pythonpkg/tests/fast/pandas/test_timestamp.py
@@ -1,8 +1,8 @@
 import duckdb
-import os
 import datetime
 import pytest
 import pandas as pd
+import platform
 from conftest import pandas_2_or_higher
 
 
@@ -64,6 +64,7 @@ def test_timestamp_timedelta(self):
         df_from_duck = duckdb.from_df(df).df()
         assert df_from_duck.equals(df)
 
+    @pytest.mark.xfail(condition=platform.system() == "Emscripten", reason="time zones not working")
     def test_timestamp_timezone(self, duckdb_cursor):
         rel = duckdb_cursor.query(
             """
diff --git a/tools/pythonpkg/tests/fast/relational_api/test_rapi_query.py b/tools/pythonpkg/tests/fast/relational_api/test_rapi_query.py
index 5fdbae2f9f64..a0141e30b5f7 100644
--- a/tools/pythonpkg/tests/fast/relational_api/test_rapi_query.py
+++ b/tools/pythonpkg/tests/fast/relational_api/test_rapi_query.py
@@ -1,5 +1,11 @@
 import duckdb
 import pytest
+import platform
+
+pytestmark = pytest.mark.skipif(
+    platform.system() == "Emscripten",
+    reason="Not supported on Emscripten",
+)
 
 
 @pytest.fixture()
diff --git a/tools/pythonpkg/tests/fast/test_alex_multithread.py b/tools/pythonpkg/tests/fast/test_alex_multithread.py
index 765b8f0ab2bf..b76745da495f 100644
--- a/tools/pythonpkg/tests/fast/test_alex_multithread.py
+++ b/tools/pythonpkg/tests/fast/test_alex_multithread.py
@@ -1,10 +1,15 @@
+import platform
 import duckdb
 from threading import Thread, current_thread
-import pandas as pd
-import os
 import pytest
 
 
+pytestmark = pytest.mark.xfail(
+    condition=platform.system() == "Emscripten",
+    reason="Emscripten builds cannot use threads",
+)
+
+
 @pytest.fixture(scope="session")
 def tmp_database(tmp_path_factory):
     database = tmp_path_factory.mktemp("databases", numbered=True) / "tmp.duckdb"
diff --git a/tools/pythonpkg/tests/fast/test_memory_leaks.py b/tools/pythonpkg/tests/fast/test_memory_leaks.py
index 228a8eace273..1d4c126603e8 100644
--- a/tools/pythonpkg/tests/fast/test_memory_leaks.py
+++ b/tools/pythonpkg/tests/fast/test_memory_leaks.py
@@ -1,9 +1,10 @@
 import gc
-import duckdb
 import pytest
-import os, psutil
+import os
 import pandas as pd
 
+psutil = pytest.importorskip("psutil")
+
 
 @pytest.fixture
 def check_leaks():
diff --git a/tools/pythonpkg/tests/fast/test_multithread.py b/tools/pythonpkg/tests/fast/test_multithread.py
index 195b1d1e4549..c4fd08372555 100644
--- a/tools/pythonpkg/tests/fast/test_multithread.py
+++ b/tools/pythonpkg/tests/fast/test_multithread.py
@@ -1,3 +1,4 @@
+import platform
 import duckdb
 import pytest
 import threading
@@ -7,12 +8,11 @@
 import os
 from typing import List
 
-try:
-    import pyarrow as pa
 
-    can_run = True
-except ImportError:
-    can_run = False
+pytestmark = pytest.mark.xfail(
+    condition=platform.system() == "Emscripten",
+    reason="Emscripten builds cannot use threads",
+)
 
 
 def connect_duck(duckdb_conn):
@@ -415,15 +415,13 @@ def test_fetchdfchunk(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_fetcharrow(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, fetch_arrow_query, pandas)
         duck_threads.multithread_test()
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_fetch_record_batch(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, fetch_record_batch_query, pandas)
         duck_threads.multithread_test()
 
@@ -449,8 +447,7 @@ def test_df_unregister(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_arrow_register_unregister(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, arrow_register_unregister, pandas)
         duck_threads.multithread_test()
 
@@ -481,8 +478,7 @@ def test_from_DF(self, duckdb_cursor, pandas):
 
     @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()])
     def test_from_arrow(self, duckdb_cursor, pandas):
-        if not can_run:
-            return
+        pytest.importorskip('pyarrow')
         duck_threads = DuckDBThreaded(10, from_arrow, pandas)
         duck_threads.multithread_test()
 
diff --git a/tools/pythonpkg/tests/fast/test_relation_dependency_leak.py b/tools/pythonpkg/tests/fast/test_relation_dependency_leak.py
index 9763d486afd2..ca5057047e52 100644
--- a/tools/pythonpkg/tests/fast/test_relation_dependency_leak.py
+++ b/tools/pythonpkg/tests/fast/test_relation_dependency_leak.py
@@ -1,6 +1,5 @@
-import duckdb
 import numpy as np
-import os, psutil
+import os
 import pytest
 
 try:
@@ -12,6 +11,9 @@
 from conftest import NumpyPandas, ArrowPandas
 
 
+psutil = pytest.importorskip("psutil")
+
+
 def check_memory(function_to_check, pandas, duckdb_cursor):
     process = psutil.Process(os.getpid())
     mem_usage = process.memory_info().rss / (10**9)