From ecb2c9f24d1364642604c14f0deb681ab4894135 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 9 Jan 2024 22:58:22 +0100 Subject: [PATCH] Set min pandas dependency to 1.2.5 for all providers and airflow (#36698) We had some REALLY old minimum version of Pandas set for all our pandas dependency - Pandas 0.17.1 has been released in 2015 (!) Looking at the dependency tree - most of our dependencies had > 1.2.5 set - which is more than reasonable limit as Pandas 1.2.5 had been released in June 2021 - so more than 2.5 years ago. This limit bump further helps us to limit the pip backtracking that starts happening in certain situations. Extracted from: #36537 --- airflow/providers/amazon/provider.yaml | 2 +- airflow/providers/apache/hive/provider.yaml | 2 +- airflow/providers/common/sql/provider.yaml | 2 +- airflow/providers/exasol/provider.yaml | 2 +- airflow/providers/google/provider.yaml | 2 +- airflow/providers/presto/provider.yaml | 2 +- airflow/providers/salesforce/provider.yaml | 2 +- airflow/providers/trino/provider.yaml | 2 +- airflow/providers/weaviate/provider.yaml | 2 +- generated/provider_dependencies.json | 14 +++++++------- setup.py | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 13c3c00bc3c70..d0f379e73df4f 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -742,7 +742,7 @@ logging: additional-extras: - name: pandas dependencies: - - pandas>=0.17.1 + - pandas>=1.2.5 # There is conflict between boto3 and aiobotocore dependency botocore. # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or # boto3 have native async support and we move away from aio aiobotocore diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 936dccc6a2ee2..d729657dc300b 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -66,7 +66,7 @@ dependencies: - apache-airflow>=2.6.0 - apache-airflow-providers-common-sql>=1.3.1 - hmsclient>=0.1.0 - - pandas>=0.17.1 + - pandas>=1.2.5 - pyhive[hive_pure_sasl]>=0.7.0 - thrift>=0.9.2 diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index 676a72727c518..dfa21587b26b8 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -54,7 +54,7 @@ dependencies: additional-extras: - name: pandas dependencies: - - pandas>=0.17.1 + - pandas>=1.2.5 integrations: - integration-name: Common SQL diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index f47d0f7d43d3c..e8488be133470 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -55,7 +55,7 @@ dependencies: - apache-airflow>=2.6.0 - apache-airflow-providers-common-sql>=1.10.0 - pyexasol>=0.5.1 - - pandas>=0.17.1 + - pandas>=1.2.5 integrations: - integration-name: Exasol diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index ed9c35637f406..aefc4d49977d6 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -137,7 +137,7 @@ dependencies: - json-merge-patch>=0.2 - looker-sdk>=22.2.0 - pandas-gbq - - pandas>=0.17.1 + - pandas>=1.2.5 # A transient dependency of google-cloud-bigquery-datatransfer, but we # further constrain it since older versions are buggy. - proto-plus>=1.19.6 diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 432354167a81f..ca47e56353a68 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -57,7 +57,7 @@ dependencies: - apache-airflow>=2.6.0 - apache-airflow-providers-common-sql>=1.3.1 - presto-python-client>=0.8.4 - - pandas>=0.17.1 + - pandas>=1.2.5 integrations: - integration-name: Presto diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index c65e83e1c2cf5..494d0ffdde2a0 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -53,7 +53,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - simple-salesforce>=1.0.0 - - pandas>=0.17.1 + - pandas>=1.2.5 integrations: - integration-name: Salesforce diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index 5542a0983f1d1..4f128048a8d67 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -57,7 +57,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - apache-airflow-providers-common-sql>=1.3.1 - - pandas>=0.17.1 + - pandas>=1.2.5 - trino>=0.318.0 integrations: diff --git a/airflow/providers/weaviate/provider.yaml b/airflow/providers/weaviate/provider.yaml index 6559c856774fc..217fb8dee5453 100644 --- a/airflow/providers/weaviate/provider.yaml +++ b/airflow/providers/weaviate/provider.yaml @@ -42,7 +42,7 @@ integrations: dependencies: - apache-airflow>=2.6.0 - weaviate-client>=3.24.2 - - pandas>=0.17.1 + - pandas>=1.2.5 hooks: - integration-name: Weaviate diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index cbbb567a52db2..76f27a032e617 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -125,7 +125,7 @@ "apache-airflow-providers-common-sql>=1.3.1", "apache-airflow>=2.6.0", "hmsclient>=0.1.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "pyhive[hive_pure_sasl]>=0.7.0", "thrift>=0.9.2" ], @@ -403,7 +403,7 @@ "deps": [ "apache-airflow-providers-common-sql>=1.10.0", "apache-airflow>=2.6.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "pyexasol>=0.5.1" ], "cross-providers-deps": [ @@ -509,7 +509,7 @@ "json-merge-patch>=0.2", "looker-sdk>=22.2.0", "pandas-gbq", - "pandas>=0.17.1", + "pandas>=1.2.5", "proto-plus>=1.19.6", "sqlalchemy-bigquery>=1.2.1", "sqlalchemy-spanner>=1.6.2" @@ -842,7 +842,7 @@ "deps": [ "apache-airflow-providers-common-sql>=1.3.1", "apache-airflow>=2.6.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "presto-python-client>=0.8.4" ], "cross-providers-deps": [ @@ -864,7 +864,7 @@ "salesforce": { "deps": [ "apache-airflow>=2.6.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "simple-salesforce>=1.0.0" ], "cross-providers-deps": [], @@ -1007,7 +1007,7 @@ "deps": [ "apache-airflow-providers-common-sql>=1.3.1", "apache-airflow>=2.6.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "trino>=0.318.0" ], "cross-providers-deps": [ @@ -1033,7 +1033,7 @@ "weaviate": { "deps": [ "apache-airflow>=2.6.0", - "pandas>=0.17.1", + "pandas>=1.2.5", "weaviate-client>=3.24.2" ], "cross-providers-deps": [], diff --git a/setup.py b/setup.py index ae3848b069e5e..1c831fb98dac7 100644 --- a/setup.py +++ b/setup.py @@ -350,7 +350,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve leveldb = ["plyvel"] otel = ["opentelemetry-exporter-prometheus"] pandas = [ - "pandas>=0.17.1", + "pandas>=1.2.5", ] password = [ "bcrypt>=2.0.0",