diff --git a/poetry.lock b/poetry.lock index 5548276..dfe26c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1067,19 +1067,19 @@ dev = ["black", "build", "mypy", "pytest", "pyupgrade", "twine", "validate-pypro [[package]] name = "delta-spark" -version = "2.4.0" +version = "3.2.0" description = "Python APIs for using Delta Lake with Apache Spark" optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "delta-spark-2.4.0.tar.gz", hash = "sha256:ef776e325e80d98e3920cab982c747b094acc46599d62dfcdc9035fb112ba6a9"}, - {file = "delta_spark-2.4.0-py3-none-any.whl", hash = "sha256:7204142a97ef16367403b020d810d0c37f4ae8275b4997de4056423cf69b3a4b"}, + {file = "delta-spark-3.2.0.tar.gz", hash = "sha256:641967828e47c64805f8c746513da80bea24b5f19b069cdcf64561cd3692e11d"}, + {file = "delta_spark-3.2.0-py3-none-any.whl", hash = "sha256:c4ff3fa7218e58a702cb71eb64384b0005c4d6f0bbdd0fe0b38a53564d946e09"}, ] [package.dependencies] importlib-metadata = ">=1.0.0" -pyspark = ">=3.4.0,<3.5.0" +pyspark = ">=3.5.0,<3.6.0" [[package]] name = "deprecated" @@ -2491,24 +2491,24 @@ extra = ["pygments (>=2.19.1)"] [[package]] name = "pyspark" -version = "3.4.4" +version = "3.5.2" description = "Apache Spark Python API" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyspark-3.4.4.tar.gz", hash = "sha256:b831a99c1aeb058bc4a2498665de10006b899b2438ba15470cc88f577755ebf8"}, + {file = "pyspark-3.5.2.tar.gz", hash = "sha256:bbb36eba09fa24e86e0923d7e7a986041b90c714e11c6aa976f9791fe9edde5e"}, ] [package.dependencies] py4j = "0.10.9.7" [package.extras] -connect = ["googleapis-common-protos (>=1.56.4)", "grpcio (>=1.48.1)", "grpcio-status (>=1.48.1)", "numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] +connect = ["googleapis-common-protos (>=1.56.4)", "grpcio (>=1.56.0)", "grpcio-status (>=1.56.0)", "numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] ml = ["numpy (>=1.15,<2)"] mllib = ["numpy (>=1.15,<2)"] -pandas-on-spark = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] -sql = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] +pandas-on-spark = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] +sql = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] [[package]] name = "pytest" @@ -3298,4 +3298,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "34d27f814106c1acab20b1dd8907f568ab59f7e7d0c0b13b087e80770b117652" +content-hash = "dc39317cbf4102e2fd18a441031ed03d8fb788fb0b54c88e6a648ab301ba1e55" diff --git a/pyproject.toml b/pyproject.toml index 4aa4d3f..59c1d2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ packages = [ python = ">=3.10,<3.13" # breaking changes beyond 3.12 boto3 = ">=1.34.162,<1.36" # breaking change beyond 1.36 botocore = ">=1.34.162,<1.36" # breaking change beyond 1.36 -delta-spark = "2.4.*" +delta-spark = "3.2.0" duckdb = "1.1.3" # breaking changes beyond 1.1 Jinja2 = "3.1.6" lxml = "4.9.4" @@ -43,7 +43,7 @@ pandas = "2.3.3" polars = "0.20.31" pyarrow = "17.0.0" pydantic = "1.10.19" -pyspark = "3.4.*" +pyspark = "3.5.2" typing_extensions = "4.15.0" [tool.poetry.group.dev] diff --git a/tests/features/patches.py b/tests/features/patches.py index 6db6a8c..c31aef4 100644 --- a/tests/features/patches.py +++ b/tests/features/patches.py @@ -89,7 +89,7 @@ def get_spark_session() -> SparkSession: os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join( [ "--packages", - "com.databricks:spark-xml_2.12:0.16.0,io.delta:delta-core_2.12:2.4.0", + "com.databricks:spark-xml_2.12:0.16.0,io.delta:delta-spark_2.12:3.2.0", "pyspark-shell", ] )