Skip to content

GH-48808: [Python] Drop support for Pandas < 2.0.3 #50444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pitrou merged 8 commits into from
Jul 9, 2026
Merged

GH-48808: [Python] Drop support for Pandas < 2.0.3 #50444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e616856
GH-48808: [Python] Investigate what entails bumping pandas >=2
raulcd Jul 9, 2026
7a04652
Drop a bunch of pd<2 cases
raulcd Jul 9, 2026
4546c60
Bump minimum Pandas CI to 2.0.0
raulcd Jul 9, 2026
e2ae2d8
Minor typo and remove unnecessary pin from pyproject.toml
raulcd Jul 9, 2026
59da22d
Remove old test only testing Pandas < 2
raulcd Jul 9, 2026
419d344
Add check for Version < 2.0.2 on Pandas test
raulcd Jul 9, 2026
9b80ede
Bump minimum pandas to 2.0.3 instead of 2.0.0
raulcd Jul 9, 2026
5dad2a2
Keep coerce_to_ns as boolean
raulcd Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/python.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
name:
- conda-python-docs
- conda-python-3.12-nopandas
- conda-python-3.11-pandas-1.5.2
- conda-python-3.11-pandas-2.0.3
- conda-python-3.14-pandas-latest
- conda-python-3.13-no-numpy
include:
Expand All @@ -79,11 +79,11 @@ jobs:
image: conda-python
title: AMD64 Conda Python 3.12 Without Pandas
python: "3.12"
- name: conda-python-3.11-pandas-1.5.2
- name: conda-python-3.11-pandas-2.0.3
image: conda-python-pandas
title: AMD64 Conda Python 3.11 Pandas 1.5.2
title: AMD64 Conda Python 3.11 Pandas 2.0.3
python: "3.11"
pandas: "1.5.2"
pandas: "2.0.3"
numpy: "1.23.2"
- name: conda-python-3.14-pandas-latest
image: conda-python-pandas
Expand Down
2 changes: 1 addition & 1 deletion dev/tasks/tasks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,7 +727,7 @@ tasks:

############################## Integration tests ############################

{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "1.5.2", "1.23.2", True),
{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.3", "1.23.2", True),
("3.12", "latest", "latest", False),
("3.13", "latest", "1.26.2", False),
("3.13", "latest", "latest", False),
Expand Down
2 changes: 1 addition & 1 deletion docs/source/python/install.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ Dependencies
Optional dependencies

* **NumPy 1.23.2** or higher.
* **pandas 1.5.2** or higher,
* **pandas 2.0.3** or higher,
* **cffi**.

Additional packages PyArrow is compatible with are :ref:`fsspec <filesystem-fsspec>`
Expand Down
4 changes: 0 additions & 4 deletions python/pyarrow/array.pxi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2383,10 +2383,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper):
arr = dtype.__from_arrow__(obj)
return pandas_api.series(arr, name=name, copy=False)

if pandas_api.is_v1():
# ARROW-3789: Coerce date/timestamp types to datetime64[ns]
c_options.coerce_temporal_nanoseconds = True

if isinstance(obj, Array):
with nogil:
check_status(ConvertArrayToPandas(c_options,
Expand Down
14 changes: 4 additions & 10 deletions python/pyarrow/pandas-shim.pxi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ cdef class _PandasAPIShim(object):
object _array_like_types, _is_extension_array_dtype, _lock
bint has_sparse
bint _pd024
bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict

def __init__(self):
self._lock = Lock()
Expand All @@ -61,25 +61,23 @@ cdef class _PandasAPIShim(object):
self._pd = pd
self._version = pd.__version__
self._loose_version = Version(pd.__version__)
self._is_v1 = False

if self._loose_version < Version('1.0.0'):
if self._loose_version < Version('2.0.3'):
self._have_pandas = False
if raise_:
raise ImportError(
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
"installed"
)
else:
warnings.warn(
f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
"installed. Therefore, pandas-specific integration is not "
"used.",
stacklevel=2
)
return

self._is_v1 = self._loose_version < Version('2.0.0')
self._is_ge_v21 = self._loose_version >= Version('2.1.0')
self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0')
self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0')
Expand Down Expand Up @@ -166,10 +164,6 @@ cdef class _PandasAPIShim(object):
self._check_import()
return self._version

def is_v1(self):
self._check_import()
return self._is_v1

def is_ge_v21(self):
self._check_import()
return self._is_ge_v21
Expand Down
2 changes: 0 additions & 2 deletions python/pyarrow/pandas_compat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,8 +790,6 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block=


def make_datetimetz(unit, tz):
if _pandas_api.is_v1():
unit = 'ns' # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
tz = pa.lib.string_to_tzinfo(tz, prefer_zoneinfo=_pandas_api.is_ge_v3())
return _pandas_api.datetimetz_type(unit, tz=tz)

Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/src/arrow/python/arrow_to_pandas.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1293,7 +1293,7 @@ struct ObjectWriterVisitor {
PyObject** out) {
ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr);
// DateOffset objects do not add nanoseconds component to pd.Timestamp.
// as of Pandas 1.3.3
// as of Pandas 1.3.3
// (https://github.com/pandas-dev/pandas/issues/43892).

@pitrou pitrou Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like pandas-dev/pandas#43892 was fixed in 1.4, can this piece of code be simplified? (though I have no idea how :))

@raulcd raulcd Jul 9, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it could be simplified to something like:

diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
index 348d352a04..581da12577 100644
--- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
+++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
@@ -1292,24 +1292,10 @@ struct ObjectWriterVisitor {
     auto to_date_offset = [&](const MonthDayNanoIntervalType::MonthDayNanos& interval,
                               PyObject** out) {
       ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr);
-      // DateOffset objects do not add nanoseconds component to pd.Timestamp.
-      // as of Pandas 1.3.3
-      // (https://github.com/pandas-dev/pandas/issues/43892).
-      // So convert microseconds and remainder to preserve data
-      // but give users more expected results.
-      int64_t microseconds = interval.nanoseconds / 1000;
-      int64_t nanoseconds;
-      if (interval.nanoseconds >= 0) {
-        nanoseconds = interval.nanoseconds % 1000;
-      } else {
-        nanoseconds = -((-interval.nanoseconds) % 1000);
-      }
 
       PyDict_SetItemString(kwargs.obj(), "months", PyLong_FromLong(interval.months));
       PyDict_SetItemString(kwargs.obj(), "days", PyLong_FromLong(interval.days));
-      PyDict_SetItemString(kwargs.obj(), "microseconds",
-                           PyLong_FromLongLong(microseconds));
-      PyDict_SetItemString(kwargs.obj(), "nanoseconds", PyLong_FromLongLong(nanoseconds));
+      PyDict_SetItemString(kwargs.obj(), "nanoseconds", PyLong_FromLongLong(interval.nanoseconds));
       *out =
           PyObject_Call(internal::BorrowPandasDataOffsetType(), args.obj(), kwargs.obj());
       RETURN_IF_PYERROR();

but I think this is a small change on the UX, nowadays we don't roundtrip hours/minutes on DateOffset we show microseconds and nanoseconds. With this change we will be consistent and we would show, Month/Day/Nano instead of Month/Day/Microseconds/Nano. I think this is worth it but maybe we should do it as a separate issue from this one around dropping older pandas supprt and discuss there whether the UX change is worth?
Are you ok if I move this to its own issue?

@pitrou pitrou Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's ok to me. Also should probably ping @jorisvandenbossche for advice :)

@raulcd raulcd Jul 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I created an issue to track this separately:

// So convert microseconds and remainder to preserve data
// but give users more expected results.
Expand Down
4 changes: 0 additions & 4 deletions python/pyarrow/table.pxi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4070,10 +4070,6 @@ def table_to_blocks(options, Table table, categories, extension_columns):
c_options.extension_columns = make_shared[unordered_set[c_string]](
unordered_set[c_string]({tobytes(col) for col in extension_columns}))

if pandas_api.is_v1():
# ARROW-3789: Coerce date/timestamp types to datetime64[ns]
c_options.coerce_temporal_nanoseconds = True

if c_options.self_destruct:
# Move the shared_ptr, table is now unsafe to use further
c_table = move(table.sp_table)
Expand Down
89 changes: 20 additions & 69 deletions python/pyarrow/tests/interchange/test_conversion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

from datetime import datetime as dt
import pyarrow as pa
from pyarrow.vendored.version import Version
import pytest

try:
Expand Down Expand Up @@ -122,9 +121,6 @@ def test_offset_of_sliced_array():
]
)
def test_pandas_roundtrip(uint, int, float, np_float_str):
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")

arr = [1, 2, 3]
table = pa.table(
{
Expand Down Expand Up @@ -153,9 +149,6 @@ def test_pandas_roundtrip(uint, int, float, np_float_str):
@pytest.mark.pandas
def test_pandas_roundtrip_string():
# See https://github.com/pandas-dev/pandas/issues/50554
if Version(pd.__version__) < Version("1.6"):
pytest.skip("Column.size() bug in pandas")

arr = ["a", "", "c"]
table = pa.table({"a": pa.array(arr)})

Expand All @@ -182,46 +175,32 @@ def test_pandas_roundtrip_string():
@pytest.mark.pandas
def test_pandas_roundtrip_large_string():
# See https://github.com/pandas-dev/pandas/issues/50554
if Version(pd.__version__) < Version("1.6"):
pytest.skip("Column.size() bug in pandas")

arr = ["a", "", "c"]
table = pa.table({"a_large": pa.array(arr, type=pa.large_string())})

from pandas.api.interchange import (
from_dataframe as pandas_from_dataframe
)

if Version(pd.__version__) >= Version("2.0.1"):
pandas_df = pandas_from_dataframe(table)
result = pi.from_dataframe(pandas_df)

assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
assert pa.types.is_large_string(table["a_large"].type)
assert pa.types.is_large_string(result["a_large"].type)
pandas_df = pandas_from_dataframe(table)
result = pi.from_dataframe(pandas_df)

table_protocol = table.__dataframe__()
result_protocol = result.__dataframe__()
assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
assert pa.types.is_large_string(table["a_large"].type)
assert pa.types.is_large_string(result["a_large"].type)

assert table_protocol.num_columns() == result_protocol.num_columns()
assert table_protocol.num_rows() == result_protocol.num_rows()
assert table_protocol.num_chunks() == result_protocol.num_chunks()
assert table_protocol.column_names() == result_protocol.column_names()
table_protocol = table.__dataframe__()
result_protocol = result.__dataframe__()

else:
# large string not supported by pandas implementation for
# older versions of pandas
# https://github.com/pandas-dev/pandas/issues/52795
with pytest.raises(AssertionError):
pandas_from_dataframe(table)
assert table_protocol.num_columns() == result_protocol.num_columns()
assert table_protocol.num_rows() == result_protocol.num_rows()
assert table_protocol.num_chunks() == result_protocol.num_chunks()
assert table_protocol.column_names() == result_protocol.column_names()


@pytest.mark.pandas
def test_pandas_roundtrip_string_with_missing():
# See https://github.com/pandas-dev/pandas/issues/50554
if Version(pd.__version__) < Version("1.6"):
pytest.skip("Column.size() bug in pandas")

arr = ["a", "", "c", None]
table = pa.table({"a": pa.array(arr),
"a_large": pa.array(arr, type=pa.large_string())})
Expand All @@ -230,29 +209,20 @@ def test_pandas_roundtrip_string_with_missing():
from_dataframe as pandas_from_dataframe
)

if Version(pd.__version__) >= Version("2.0.2"):
pandas_df = pandas_from_dataframe(table)
result = pi.from_dataframe(pandas_df)
pandas_df = pandas_from_dataframe(table)
result = pi.from_dataframe(pandas_df)

assert result["a"].to_pylist() == table["a"].to_pylist()
assert pa.types.is_string(table["a"].type)
assert pa.types.is_large_string(result["a"].type)
assert result["a"].to_pylist() == table["a"].to_pylist()
assert pa.types.is_string(table["a"].type)
assert pa.types.is_large_string(result["a"].type)

assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
assert pa.types.is_large_string(table["a_large"].type)
assert pa.types.is_large_string(result["a_large"].type)
else:
# older versions of pandas do not have bitmask support
# https://github.com/pandas-dev/pandas/issues/49888
with pytest.raises(NotImplementedError):
pandas_from_dataframe(table)
assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
assert pa.types.is_large_string(table["a_large"].type)
assert pa.types.is_large_string(result["a_large"].type)


@pytest.mark.pandas
def test_pandas_roundtrip_categorical():
if Version(pd.__version__) < Version("2.0.2"):
pytest.skip("Bitmasks not supported in pandas interchange implementation")

arr = ["Mon", "Tue", "Mon", "Wed", "Mon", "Thu", "Fri", "Sat", None]
table = pa.table(
{"weekday": pa.array(arr).dictionary_encode()}
Expand Down Expand Up @@ -299,21 +269,14 @@ def test_pandas_roundtrip_categorical():
@pytest.mark.pandas
@pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns'])
def test_pandas_roundtrip_datetime(unit):
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")
from datetime import datetime as dt

# timezones not included as they are not yet supported in
# the pandas implementation
dt_arr = [dt(2007, 7, 13), dt(2007, 7, 14), dt(2007, 7, 15)]
table = pa.table({"a": pa.array(dt_arr, type=pa.timestamp(unit))})

if Version(pd.__version__) < Version("1.6"):
# pandas < 2.0 always creates datetime64 in "ns"
# resolution
expected = pa.table({"a": pa.array(dt_arr, type=pa.timestamp('ns'))})
else:
expected = table
expected = table

from pandas.api.interchange import (
from_dataframe as pandas_from_dataframe
Expand All @@ -337,9 +300,6 @@ def test_pandas_roundtrip_datetime(unit):
"np_float_str", ["float32", "float64"]
)
def test_pandas_to_pyarrow_with_missing(np_float_str):
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")

np_array = np.array([0, np.nan, 2], dtype=np.dtype(np_float_str))
datetime_array = [None, dt(2007, 7, 14), dt(2007, 7, 15)]
df = pd.DataFrame({
Expand All @@ -359,9 +319,6 @@ def test_pandas_to_pyarrow_with_missing(np_float_str):

@pytest.mark.pandas
def test_pandas_to_pyarrow_float16_with_missing():
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")

# np.float16 errors if ps.is_nan is used
# pyarrow.lib.ArrowNotImplementedError: Function 'is_nan' has no kernel
# matching input types (halffloat)
Expand Down Expand Up @@ -482,9 +439,6 @@ def test_nan_as_null():

@pytest.mark.pandas
def test_allow_copy_false():
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")

# Test that an error is raised when a copy is needed
# to create a bitmask

Expand All @@ -501,9 +455,6 @@ def test_allow_copy_false():

@pytest.mark.pandas
def test_allow_copy_false_bool_categorical():
if Version(pd.__version__) < Version("1.5.0"):
pytest.skip("__dataframe__ added to pandas in 1.5.0")

# Test that an error is raised for boolean
# and categorical dtype (copy is always made)

Expand Down
3 changes: 0 additions & 3 deletions python/pyarrow/tests/parquet/test_pandas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,9 +524,6 @@ def test_pandas_categorical_roundtrip():
@pytest.mark.pandas
def test_categories_with_string_pyarrow_dtype(tempdir):
# gh-33727: writing to parquet should not fail
if Version(pd.__version__) < Version("1.3.0"):
pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0")

df1 = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]")
df1 = df1.astype("category")

Expand Down
18 changes: 5 additions & 13 deletions python/pyarrow/tests/test_compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2612,8 +2612,6 @@ def test_strftime():


def _check_datetime_components(timestamps, timezone=None):
from pyarrow.vendored.version import Version

ts = pd.to_datetime(timestamps).tz_localize(
"UTC").tz_convert(timezone).to_series()
tsa = pa.array(ts, pa.timestamp("ns", tz=timezone))
Expand All @@ -2626,17 +2624,11 @@ def _check_datetime_components(timestamps, timezone=None):
pa.field('iso_day_of_week', pa.int64())
]

if Version(pd.__version__) < Version("1.1.0"):
# https://github.com/pandas-dev/pandas/issues/33206
iso_year = ts.map(lambda x: x.isocalendar()[0]).astype("int64")
iso_week = ts.map(lambda x: x.isocalendar()[1]).astype("int64")
iso_day = ts.map(lambda x: x.isocalendar()[2]).astype("int64")
else:
# Casting is required because pandas isocalendar returns int32
# while arrow isocalendar returns int64.
iso_year = ts.dt.isocalendar()["year"].astype("int64")
iso_week = ts.dt.isocalendar()["week"].astype("int64")
iso_day = ts.dt.isocalendar()["day"].astype("int64")
# Casting is required because pandas isocalendar returns int32
# while arrow isocalendar returns int64.
iso_year = ts.dt.isocalendar()["year"].astype("int64")
iso_week = ts.dt.isocalendar()["week"].astype("int64")
iso_day = ts.dt.isocalendar()["day"].astype("int64")

iso_calendar = pa.StructArray.from_arrays(
[iso_year, iso_week, iso_day],
Expand Down
Loading
Loading