refactor!: Remove explicit pyarrow dependency

dataframely/_compat.py

@@ -52,13 +52,6 @@ class PGDialect_psycopg2:  # type: ignore # noqa: N801
     class Dialect:  # type: ignore # noqa: N801
         pass
 
-# -------------------------------------- PYARROW ------------------------------------- #
-
-try:
-    import pyarrow as pa
-except ImportError:  # pragma: no cover
-    pa = _DummyModule("pyarrow")  # type: ignore
-
 # -------------------------------------- PYDANTIC ------------------------------------ #
 
 try:
@@ -97,7 +90,6 @@ class Dialect:  # type: ignore # noqa: N801
     "Dialect",
     "MSDialect_pyodbc",
     "PartitionSchemeOrSinkDirectory",
-    "pa",
     "PGDialect_psycopg2",
     "pydantic_core_schema",
     "pydantic",

dataframely/columns/_base.py

@@ -13,7 +13,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, pydantic, sa, sa_TypeEngine
+from dataframely._compat import pydantic, sa, sa_TypeEngine
 from dataframely._polars import PolarsDataType
 from dataframely.random import Generator
 
@@ -224,24 +224,6 @@ def sqlalchemy_column(self, name: str, dialect: sa.Dialect) -> sa.Column:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         """The :mod:`sqlalchemy` dtype equivalent of this column data type."""
 
-    # ------------------------------------ PYARROW ----------------------------------- #
-
-    def pyarrow_field(self, name: str) -> pa.Field:
-        """Obtain the pyarrow field of this column definition.
-
-        Args:
-            name: The name of the column.
-
-        Returns:
-            The :mod:`pyarrow` field definition.
-        """
-        return pa.field(name, self.pyarrow_dtype, nullable=self.nullable)
-
-    @property
-    @abstractmethod
-    def pyarrow_dtype(self) -> pa.DataType:
-        """The :mod:`pyarrow` dtype equivalent of this column data type."""
-
     # ----------------------------------- PYDANTIC ----------------------------------- #
 
     def pydantic_field(self) -> Any:

dataframely/columns/any.py

@@ -7,7 +7,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_mssql, sa_TypeEngine
+from dataframely._compat import sa, sa_mssql, sa_TypeEngine
 from dataframely._polars import PolarsDataType
 from dataframely.random import Generator
 
@@ -79,13 +79,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:  # pragma: no cover
                 raise NotImplementedError("SQL column cannot have 'Any' type.")
 
-    def pyarrow_field(self, name: str) -> pa.Field:
-        return pa.field(name, self.pyarrow_dtype, nullable=self.nullable)
-
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.null()
-
     @property
     def _python_type(self) -> AnyType:
         return AnyType

dataframely/columns/array.py

@@ -6,12 +6,11 @@
 import math
 import sys
 import warnings
-from collections.abc import Sequence
 from typing import Any, cast
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely.random import Generator
 
 from ._base import Check, Column
@@ -132,18 +131,6 @@ def _arrow_nullability(self) -> tuple[bool, list[Any]]:
             nullability = (True, [nullability])
         return (self.nullable, [nullability])
 
-    def _pyarrow_field_of_shape(self, shape: Sequence[int]) -> pa.Field:
-        if shape:
-            size, *rest = shape
-            inner_type = self._pyarrow_field_of_shape(rest)
-            return pa.field("item", pa.list_(inner_type, size), nullable=True)
-        else:
-            return self.inner.pyarrow_field("item")
-
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return self._pyarrow_field_of_shape(self.shape).type
-
     @property
     def _python_type(self) -> Any:
         inner_type = self.inner.pydantic_field()

dataframely/columns/binary.py

@@ -7,7 +7,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely.random import Generator
 
 from ._base import Column
@@ -29,10 +29,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:
                 return sa.LargeBinary()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.large_binary()
-
     @property
     def _python_type(self) -> Any:
         return bytes

dataframely/columns/bool.py

@@ -7,7 +7,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely.random import Generator
 
 from ._base import Column
@@ -25,10 +25,6 @@ def dtype(self) -> pl.DataType:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         return sa.Boolean()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.bool_()
-
     @property
     def _python_type(self) -> Any:
         return bool

dataframely/columns/categorical.py

@@ -7,7 +7,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely.random import Generator
 
 from ._base import Check, Column
@@ -79,10 +79,6 @@ def dtype(self) -> pl.DataType:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         return sa.String()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.dictionary(pa.uint32(), pa.large_string())
-
     @property
     def _python_type(self) -> Any:
         return str

dataframely/columns/datetime.py

@@ -10,7 +10,7 @@
 import polars as pl
 from polars._typing import TimeUnit
 
-from dataframely._compat import pa, sa, sa_mssql, sa_TypeEngine
+from dataframely._compat import sa, sa_mssql, sa_TypeEngine
 from dataframely._polars import (
     EPOCH_DATETIME,
     date_matches_resolution,
@@ -141,10 +141,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:
                 return sa.Date()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.date32()
-
     @property
     def _python_type(self) -> Any:
         return dt.date
@@ -292,10 +288,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:
                 return sa.Time()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.time64("ns")
-
     @property
     def _python_type(self) -> Any:
         return dt.time
@@ -442,15 +434,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:
                 return sa.DateTime(timezone=timezone_enabled)
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        time_zone = (
-            self.time_zone.tzname(None)
-            if isinstance(self.time_zone, dt.tzinfo)
-            else self.time_zone
-        )
-        return pa.timestamp(self.time_unit, time_zone)
-
     @property
     def _python_type(self) -> Any:
         return dt.datetime
@@ -612,10 +595,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             case _:
                 return sa.Interval()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.duration(self.time_unit)
-
     @property
     def _python_type(self) -> Any:
         return dt.timedelta

dataframely/columns/decimal.py

@@ -8,7 +8,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely._polars import PolarsDataType
 from dataframely.random import Generator
 
@@ -133,13 +133,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         else:
             return sa.Numeric(self.precision, self.scale)
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        # PyArrow requires an explicit value for precision.
-        # If precision is None, we pass decimal128's maximum precision of 38 to be safe.
-        # We do not use decimal256 since its values cannot be represented in SQL Server.
-        return pa.decimal128(self.precision or 38, self.scale)
-
     @property
     def _python_type(self) -> Any:
         return decimal.Decimal

dataframely/columns/enum.py

@@ -10,7 +10,7 @@
 
 import polars as pl
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely._polars import PolarsDataType
 from dataframely.random import Generator
 
@@ -130,16 +130,6 @@ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
             return sa.CHAR(category_lengths[0])
         return sa.String(max(category_lengths))
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        if len(self.categories) <= 2**8 - 1:
-            dtype = pa.uint8()
-        elif len(self.categories) <= 2**16 - 1:
-            dtype = pa.uint16()
-        else:
-            dtype = pa.uint32()
-        return pa.dictionary(dtype, pa.large_string(), ordered=True)
-
     @property
     def _python_type(self) -> Any:
         return Literal[tuple(self.categories)]

dataframely/columns/float.py

@@ -13,7 +13,7 @@
 import polars as pl
 from polars.datatypes.group import FLOAT_DTYPES
 
-from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely._compat import sa, sa_TypeEngine
 from dataframely._polars import PolarsDataType
 from dataframely.random import Generator
 
@@ -193,10 +193,6 @@ def validate_dtype(self, dtype: PolarsDataType) -> bool:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         return sa.Float()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.float64()
-
     @classproperty
     def max_value(self) -> float:
         return float(np.finfo(np.float64).max)
@@ -217,10 +213,6 @@ def dtype(self) -> pl.DataType:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         return sa.REAL()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.float32()
-
     @classproperty
     def max_value(self) -> float:
         return float(np.finfo(np.float32).max)
@@ -241,10 +233,6 @@ def dtype(self) -> pl.DataType:
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         return sa.Float()
 
-    @property
-    def pyarrow_dtype(self) -> pa.DataType:
-        return pa.float64()
-
     @classproperty
     def max_value(self) -> float:
         return float(np.finfo(np.float64).max)