[Python] pandas is_sparse attribute is deprecated

[jorisvandenbossche]

We see warnings in the nightly logs:

 opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/parquet/test_metadata.py: 13 warnings
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/parquet/test_pandas.py: 4 warnings
  /opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/pandas_compat.py:456: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.
    if _pandas_api.is_sparse(col):

and that also causes two actual test failures because we are ensuring some code didn't trigger any warning

 _______________ TestConvertMetadata.test_rangeindex_doesnt_warn ________________

self = <pyarrow.tests.test_pandas.TestConvertMetadata object at 0x7f0969e30b40>

    def test_rangeindex_doesnt_warn(self):
        # ARROW-5606: pandas 0.25 deprecated private _start/stop/step
        # attributes -> can be removed if support < pd 0.25 is dropped
        df = pd.DataFrame(np.random.randn(4, 2), columns=['a', 'b'])
    
        with warnings.catch_warnings():
            warnings.simplefilter(action="error")
>           _check_pandas_roundtrip(df, preserve_index=True)

opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/test_pandas.py:247: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/test_pandas.py:86: in _check_pandas_roundtrip
    table = klass.from_pandas(df, schema=schema,
pyarrow/table.pxi:3681: in pyarrow.lib.Table.from_pandas
    ???
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/pandas_compat.py:570: in dataframe_to_arrays
    convert_fields) = _get_columns_to_convert(df, schema, preserve_index,
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/pandas_compat.py:373: in _get_columns_to_convert
    if _pandas_api.is_sparse(col):
pyarrow/pandas-shim.pxi:191: in pyarrow.lib._PandasAPIShim.is_sparse
    ???
pyarrow/pandas-shim.pxi:193: in pyarrow.lib._PandasAPIShim.is_sparse
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

arr = 0   -0.299318
1    0.115113
2    2.195990
3    1.032101
Name: a, dtype: float64

    def is_sparse(arr) -> bool:
        """
        Check whether an array-like is a 1-D pandas sparse array.
    
        Check that the one-dimensional array-like is a pandas sparse array.
        Returns True if it is a pandas sparse array, not another type of
        sparse array.
    
        Parameters
        ----------
        arr : array-like
            Array-like to check.
    
        Returns
        -------
        bool
            Whether or not the array-like is a pandas sparse array.
    
        Examples
        --------
        Returns `True` if the parameter is a 1-D pandas sparse array.
    
        >>> from pandas.api.types import is_sparse
        >>> is_sparse(pd.arrays.SparseArray([0, 0, 1, 0]))
        True
        >>> is_sparse(pd.Series(pd.arrays.SparseArray([0, 0, 1, 0])))
        True
    
        Returns `False` if the parameter is not sparse.
    
        >>> is_sparse(np.array([0, 0, 1, 0]))
        False
        >>> is_sparse(pd.Series([0, 1, 0, 0]))
        False
    
        Returns `False` if the parameter is not a pandas sparse array.
    
        >>> from scipy.sparse import bsr_matrix
        >>> is_sparse(bsr_matrix([0, 1, 0, 0]))
        False
    
        Returns `False` if the parameter has more than one dimension.
        """
>       warnings.warn(
            "is_sparse is deprecated and will be removed in a future "
            "version. Check `isinstance(dtype, pd.SparseDtype)` instead.",
            FutureWarning,
            stacklevel=find_stack_level(),
        )
E       FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.

opt/conda/envs/arrow/lib/python3.9/site-packages/pandas/core/dtypes/common.py:210: FutureWarning
_______________ TestConvertMetadata.test_multiindex_doesnt_warn ________________

self = <pyarrow.tests.test_pandas.TestConvertMetadata object at 0x7f09687cea50>

    def test_multiindex_doesnt_warn(self):
        # ARROW-3953: pandas 0.24 rename of MultiIndex labels to codes
        columns = pd.MultiIndex.from_arrays([['one', 'two'], ['X', 'Y']])
        df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns)
    
        with warnings.catch_warnings():
            warnings.simplefilter(action="error")
>           _check_pandas_roundtrip(df, preserve_index=True)

opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/test_pandas.py:297: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/tests/test_pandas.py:86: in _check_pandas_roundtrip
    table = klass.from_pandas(df, schema=schema,
pyarrow/table.pxi:3681: in pyarrow.lib.Table.from_pandas
    ???
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/pandas_compat.py:570: in dataframe_to_arrays
    convert_fields) = _get_columns_to_convert(df, schema, preserve_index,
opt/conda/envs/arrow/lib/python3.9/site-packages/pyarrow/pandas_compat.py:373: in _get_columns_to_convert
    if _pandas_api.is_sparse(col):
pyarrow/pandas-shim.pxi:191: in pyarrow.lib._PandasAPIShim.is_sparse
    ???
pyarrow/pandas-shim.pxi:193: in pyarrow.lib._PandasAPIShim.is_sparse
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

arr = 0    1
1    2
2    3
Name: (one, X), dtype: int64

    def is_sparse(arr) -> bool:
        """
        Check whether an array-like is a 1-D pandas sparse array.
    
        Check that the one-dimensional array-like is a pandas sparse array.
        Returns True if it is a pandas sparse array, not another type of
        sparse array.
    
        Parameters
        ----------
        arr : array-like
            Array-like to check.
    
        Returns
        -------
        bool
            Whether or not the array-like is a pandas sparse array.
    
        Examples
        --------
        Returns `True` if the parameter is a 1-D pandas sparse array.
    
        >>> from pandas.api.types import is_sparse
        >>> is_sparse(pd.arrays.SparseArray([0, 0, 1, 0]))
        True
        >>> is_sparse(pd.Series(pd.arrays.SparseArray([0, 0, 1, 0])))
        True
    
        Returns `False` if the parameter is not sparse.
    
        >>> is_sparse(np.array([0, 0, 1, 0]))
        False
        >>> is_sparse(pd.Series([0, 1, 0, 0]))
        False
    
        Returns `False` if the parameter is not a pandas sparse array.
    
        >>> from scipy.sparse import bsr_matrix
        >>> is_sparse(bsr_matrix([0, 1, 0, 0]))
        False
    
        Returns `False` if the parameter has more than one dimension.
        """
>       warnings.warn(
            "is_sparse is deprecated and will be removed in a future "
            "version. Check `isinstance(dtype, pd.SparseDtype)` instead.",
            FutureWarning,
            stacklevel=find_stack_level(),
        )
E       FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.