From 42ad802e9f4ce0b95b22b9e50cec00d15b4b1149 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 12:50:40 +0100 Subject: [PATCH 1/6] fix(airflow): rename constraints file --- ...straints-2.8.1-python39.txt => constraints-2.8.3-python39.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename airflow/{constraints-2.8.1-python39.txt => constraints-2.8.3-python39.txt} (100%) diff --git a/airflow/constraints-2.8.1-python39.txt b/airflow/constraints-2.8.3-python39.txt similarity index 100% rename from airflow/constraints-2.8.1-python39.txt rename to airflow/constraints-2.8.3-python39.txt From 5d78367df7cf5baf517280c7e8114a422594ac6f Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 13:45:22 +0100 Subject: [PATCH 2/6] feat(airflow): add helper script for downloading the constraint files --- airflow/README.md | 20 ++++++++++++++++++-- airflow/download_constraints.sh | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100755 airflow/download_constraints.sh diff --git a/airflow/README.md b/airflow/README.md index 016d67d4b..221841c9a 100644 --- a/airflow/README.md +++ b/airflow/README.md @@ -1,5 +1,21 @@ # Airflow constraints -The constraints file can be downloaded from where `2.7.2` is the airflow version and `3.9` is the python version. +To add a new version of airflow (or python) you can download the -Due to the build in the docker image, the python version must be specified without the `dot`. E.g. `3.9` must be renamed to `39` in the constraints file name. +_The script is safe to run from outside of this directory. The file will be downloaded to the same folder that the +script resides. This example will assume you are running from the repository root directory._ + +```sh +# Use default Python version (specified in the script): +./airflow/download_constraints.sh 2.8.3 + +# Use a specific Python version: +./airflow/download_constraints.sh 2.8.3 3.11 +``` + +Example output: + +``` +Downloading constraints file for Airflow 2.8.3 (Python 3.9) +Successfully pulled new constraints file: constraints-2.8.3-python39.txt +``` diff --git a/airflow/download_constraints.sh b/airflow/download_constraints.sh new file mode 100755 index 000000000..3eff91225 --- /dev/null +++ b/airflow/download_constraints.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +AIRFLOW_VERSION=${1:?"Missing version number argument (arg 1)"} +PYTHON_VERSION=${2:-"3.9"} + +URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" +FILENAME="constraints-${AIRFLOW_VERSION}-python${PYTHON_VERSION//.}.txt" + +# Find the directory name of the script so it can be run from outside +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +pushd ${DIR} > /dev/null + +echo "Downloading constraints file for Airflow ${AIRFLOW_VERSION} (Python ${PYTHON_VERSION})" +curl --fail -Ls "${URL}" -o "${FILENAME}" + +echo "Successfully pulled new constraints file: ${FILENAME}" +popd > /dev/null From 6a9619fd249449bc2edf91158ce14d54c2207cc4 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 13:45:48 +0100 Subject: [PATCH 3/6] fix(airflow): us upstream constraints file --- airflow/constraints-2.8.3-python39.txt | 583 +++++++++++++------------ 1 file changed, 292 insertions(+), 291 deletions(-) diff --git a/airflow/constraints-2.8.3-python39.txt b/airflow/constraints-2.8.3-python39.txt index 430908024..b0336c7a8 100644 --- a/airflow/constraints-2.8.3-python39.txt +++ b/airflow/constraints-2.8.3-python39.txt @@ -1,5 +1,6 @@ + # -# This constraints file was automatically generated on 2024-01-16T07:06:10.316386 +# This constraints file was automatically generated on 2024-03-12T17:14:14.904649 # via "eager-upgrade" mechanism of PIP. For the "v2-8-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. @@ -33,26 +34,27 @@ Authlib==1.3.0 Babel==2.14.0 ConfigUpdater==3.2 Deprecated==1.2.14 -Flask-AppBuilder==4.3.10 +Flask-AppBuilder==4.3.11 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 Flask-Caching==2.1.0 Flask-JWT-Extended==4.6.0 -Flask-Limiter==3.5.0 +Flask-Limiter==3.5.1 Flask-Login==0.6.3 Flask-SQLAlchemy==2.5.1 Flask-Session==0.5.0 Flask-WTF==1.2.1 Flask==2.2.5 -GitPython==3.1.41 +GitPython==3.1.42 JPype1==1.5.0 JayDeBeApi==1.2.3 Jinja2==3.1.3 Js2Py==0.74 -Mako==1.3.0 +Mako==1.3.2 Markdown==3.5.2 -MarkupSafe==2.1.3 -PyGithub==2.1.1 +MarkupSafe==2.1.5 +PyAthena==3.3.0 +PyGithub==2.2.0 PyHive==0.7.0 PyJWT==2.8.0 PyNaCl==1.5.0 @@ -60,181 +62,182 @@ PyYAML==6.0.1 Pygments==2.17.2 SQLAlchemy-JSONField==1.0.2 SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.51 +SQLAlchemy==1.4.52 SecretStorage==3.3.3 Sphinx==5.3.0 WTForms==3.1.2 Werkzeug==2.2.3 adal==1.2.7 -adlfs==2023.12.0 -aiobotocore==2.9.0 +adlfs==2024.2.0 +aiobotocore==2.11.2 aiofiles==23.2.1 -aiohttp==3.9.1 +aiohttp==3.9.3 aioitertools==0.11.0 aioresponses==0.7.6 aiosignal==1.3.1 alabaster==0.7.16 alembic==1.13.1 -alibabacloud-adb20211201==1.2.6 +alibabacloud-adb20211201==1.2.9 +alibabacloud-credentials==0.3.2 +alibabacloud-endpoint-util==0.0.3 +alibabacloud-gateway-spi==0.0.1 +alibabacloud-openapi-util==0.2.2 +alibabacloud-tea-openapi==0.3.8 +alibabacloud-tea-util==0.3.11 +alibabacloud-tea-xml==0.0.2 alibabacloud-tea==0.3.5 -alibabacloud_credentials==0.3.2 -alibabacloud_endpoint_util==0.0.3 -alibabacloud_gateway_spi==0.0.1 -alibabacloud_openapi_util==0.2.2 -alibabacloud_tea_openapi==0.3.8 -alibabacloud_tea_util==0.3.11 -alibabacloud_tea_xml==0.0.2 -aliyun-python-sdk-core==2.14.0 +aliyun-python-sdk-core==2.15.0 aliyun-python-sdk-kms==2.16.2 amqp==5.2.0 analytics-python==1.2.9 annotated-types==0.6.0 anyascii==0.3.2 -anyio==4.2.0 -apache-airflow-providers-airbyte==3.5.1 -apache-airflow-providers-alibaba==2.7.1 -apache-airflow-providers-amazon==8.16.0 -apache-airflow-providers-apache-beam==5.5.0 +anyio==4.3.0 +apache-airflow-providers-airbyte==3.6.0 +apache-airflow-providers-alibaba==2.7.2 +apache-airflow-providers-amazon==8.19.0 +apache-airflow-providers-apache-beam==5.6.2 apache-airflow-providers-apache-cassandra==3.4.1 -apache-airflow-providers-apache-drill==2.6.0 -apache-airflow-providers-apache-druid==3.7.0 +apache-airflow-providers-apache-drill==2.6.1 +apache-airflow-providers-apache-druid==3.9.0 apache-airflow-providers-apache-flink==1.3.0 -apache-airflow-providers-apache-hdfs==4.3.2 -apache-airflow-providers-apache-hive==6.4.1 +apache-airflow-providers-apache-hdfs==4.3.3 +apache-airflow-providers-apache-hive==7.0.1 apache-airflow-providers-apache-impala==1.3.0 apache-airflow-providers-apache-kafka==1.3.1 apache-airflow-providers-apache-kylin==3.5.0 -apache-airflow-providers-apache-livy==3.7.1 +apache-airflow-providers-apache-livy==3.7.3 apache-airflow-providers-apache-pig==4.3.0 -apache-airflow-providers-apache-pinot==4.3.0 -apache-airflow-providers-apache-spark==4.7.0 -apache-airflow-providers-apprise==1.2.1 +apache-airflow-providers-apache-pinot==4.3.1 +apache-airflow-providers-apache-spark==4.7.1 +apache-airflow-providers-apprise==1.2.2 apache-airflow-providers-arangodb==2.4.1 apache-airflow-providers-asana==2.4.1 -apache-airflow-providers-atlassian-jira==2.5.0 -apache-airflow-providers-celery==3.5.1 +apache-airflow-providers-atlassian-jira==2.5.1 +apache-airflow-providers-celery==3.6.1 apache-airflow-providers-cloudant==3.4.1 -apache-airflow-providers-cncf-kubernetes==7.13.0 -apache-airflow-providers-cohere==1.1.1 -apache-airflow-providers-common-io==1.2.0 -apache-airflow-providers-common-sql==1.10.0 -apache-airflow-providers-databricks==6.0.0 +apache-airflow-providers-cncf-kubernetes==8.0.1 +apache-airflow-providers-cohere==1.1.2 +apache-airflow-providers-common-io==1.3.0 +apache-airflow-providers-common-sql==1.11.1 +apache-airflow-providers-databricks==6.2.0 apache-airflow-providers-datadog==3.5.1 -apache-airflow-providers-dbt-cloud==3.5.1 +apache-airflow-providers-dbt-cloud==3.7.0 apache-airflow-providers-dingding==3.4.0 -apache-airflow-providers-discord==3.5.0 -apache-airflow-providers-docker==3.9.1 -apache-airflow-providers-elasticsearch==5.3.1 -apache-airflow-providers-exasol==4.4.1 +apache-airflow-providers-discord==3.6.0 +apache-airflow-providers-docker==3.9.2 +apache-airflow-providers-elasticsearch==5.3.3 +apache-airflow-providers-exasol==4.4.3 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1 -apache-airflow-providers-google==10.13.1 +apache-airflow-providers-google==10.16.0 apache-airflow-providers-grpc==3.4.1 -apache-airflow-providers-hashicorp==3.6.1 -apache-airflow-providers-http==4.8.0 +apache-airflow-providers-hashicorp==3.6.4 +apache-airflow-providers-http==4.10.0 apache-airflow-providers-imap==3.5.0 -apache-airflow-providers-influxdb==2.4.0 -apache-airflow-providers-jdbc==4.2.1 +apache-airflow-providers-influxdb==2.4.1 +apache-airflow-providers-jdbc==4.2.2 apache-airflow-providers-jenkins==3.5.1 -apache-airflow-providers-microsoft-azure==8.5.1 -apache-airflow-providers-microsoft-mssql==3.6.0 -apache-airflow-providers-microsoft-psrp==2.5.0 +apache-airflow-providers-microsoft-azure==9.0.1 +apache-airflow-providers-microsoft-mssql==3.6.1 +apache-airflow-providers-microsoft-psrp==2.6.0 apache-airflow-providers-microsoft-winrm==3.4.0 -apache-airflow-providers-mongo==3.5.0 -apache-airflow-providers-mysql==5.5.1 +apache-airflow-providers-mongo==4.0.0 +apache-airflow-providers-mysql==5.5.4 apache-airflow-providers-neo4j==3.5.0 -apache-airflow-providers-odbc==4.4.0 +apache-airflow-providers-odbc==4.4.1 apache-airflow-providers-openai==1.1.0 apache-airflow-providers-openfaas==3.4.0 -apache-airflow-providers-openlineage==1.4.0 -apache-airflow-providers-opensearch==1.1.1 +apache-airflow-providers-openlineage==1.6.0 +apache-airflow-providers-opensearch==1.1.2 apache-airflow-providers-opsgenie==5.5.0 -apache-airflow-providers-oracle==3.9.1 -apache-airflow-providers-pagerduty==3.6.0 -apache-airflow-providers-papermill==3.6.0 +apache-airflow-providers-oracle==3.9.2 +apache-airflow-providers-pagerduty==3.6.2 +apache-airflow-providers-papermill==3.6.1 apache-airflow-providers-pgvector==1.1.0 -apache-airflow-providers-pinecone==1.1.1 -apache-airflow-providers-postgres==5.10.0 -apache-airflow-providers-presto==5.4.0 +apache-airflow-providers-pinecone==1.1.2 +apache-airflow-providers-postgres==5.10.2 +apache-airflow-providers-presto==5.4.2 apache-airflow-providers-redis==3.6.0 -apache-airflow-providers-salesforce==5.6.1 +apache-airflow-providers-salesforce==5.6.3 apache-airflow-providers-samba==4.5.0 apache-airflow-providers-segment==3.4.0 apache-airflow-providers-sendgrid==3.4.0 -apache-airflow-providers-sftp==4.8.1 +apache-airflow-providers-sftp==4.9.0 apache-airflow-providers-singularity==3.4.0 -apache-airflow-providers-slack==8.5.1 -apache-airflow-providers-smtp==1.6.0 -apache-airflow-providers-snowflake==5.2.1 -apache-airflow-providers-sqlite==3.7.0 -apache-airflow-providers-ssh==3.10.0 -apache-airflow-providers-tableau==4.4.0 +apache-airflow-providers-slack==8.6.1 +apache-airflow-providers-smtp==1.6.1 +apache-airflow-providers-snowflake==5.3.1 +apache-airflow-providers-sqlite==3.7.1 +apache-airflow-providers-ssh==3.10.1 +apache-airflow-providers-tableau==4.4.2 apache-airflow-providers-tabular==1.4.1 -apache-airflow-providers-telegram==4.3.0 -apache-airflow-providers-trino==5.6.0 -apache-airflow-providers-vertica==3.7.0 -apache-airflow-providers-weaviate==1.3.0 -apache-airflow-providers-yandex==3.7.1 +apache-airflow-providers-telegram==4.4.0 +apache-airflow-providers-trino==5.6.3 +apache-airflow-providers-vertica==3.7.1 +apache-airflow-providers-weaviate==1.3.3 +apache-airflow-providers-yandex==3.9.0 apache-airflow-providers-zendesk==4.6.0 -apache-beam==2.53.0 -apispec==6.4.0 -apprise==1.7.1 -argcomplete==3.2.1 +apache-beam==2.54.0 +apispec==6.5.0 +apprise==1.7.4 +argcomplete==3.2.3 asana==3.2.2 asgiref==3.7.2 asn1crypto==1.5.1 astroid==2.15.8 asttokens==2.4.1 async-timeout==4.0.3 +asyncssh==2.14.2 atlasclient==1.0.0 -atlassian-python-api==3.41.5 +atlassian-python-api==3.41.11 attrs==23.2.0 -aws-sam-translator==1.83.0 -aws-xray-sdk==2.12.1 +aws-sam-translator==1.86.0 +aws-xray-sdk==2.13.0 azure-batch==14.1.0 azure-common==1.1.28 -azure-core==1.29.6 +azure-core==1.30.1 azure-cosmos==4.5.1 azure-datalake-store==0.0.53 azure-identity==1.15.0 -azure-keyvault-secrets==4.7.0 +azure-keyvault-secrets==4.8.0 azure-kusto-data==4.3.1 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==10.3.0 azure-mgmt-core==1.4.0 azure-mgmt-cosmosdb==9.4.0 -azure-mgmt-datafactory==4.0.0 +azure-mgmt-datafactory==6.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-nspkg==3.0.2 -azure-servicebus==7.11.4 -azure-storage-blob==12.19.0 +azure-servicebus==7.12.0 +azure-storage-blob==12.19.1 azure-storage-file-datalake==12.14.0 azure-storage-file-share==12.15.0 azure-synapse-artifacts==0.18.0 azure-synapse-spark==0.7.0 backoff==2.2.1 bcrypt==4.1.2 -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 billiard==4.2.0 bitarray==2.9.2 -black==24.1a1 +black==24.2.0 blinker==1.7.0 boto3==1.33.13 botocore==1.33.13 cachelib==0.9.0 -cachetools==5.3.2 +cachetools==5.3.3 cassandra-driver==3.29.0 cattrs==23.2.3 celery==5.3.6 -certifi==2023.11.17 +certifi==2024.2.2 cffi==1.16.0 cfgv==3.4.0 -cfn-lint==0.83.8 +cfn-lint==0.86.0 cgroupspy==0.2.2 chardet==5.2.0 charset-normalizer==3.3.2 @@ -247,126 +250,126 @@ click==8.1.7 clickclick==20.10.2 cloudant==2.15.0 cloudpickle==2.2.1 -cohere==4.44 +cohere==4.54 colorama==0.4.6 colorlog==4.8.0 -comm==0.2.1 +comm==0.2.2 confluent-kafka==2.3.0 connexion==2.14.2 -coverage==7.4.0 +coverage==7.4.3 crcmod==1.7 -cron-descriptor==1.4.0 -croniter==2.0.1 +cron-descriptor==1.4.3 +croniter==2.0.2 cryptography==41.0.7 curlify==2.2.1 -databricks-sql-connector==2.9.3 -datadog==0.47.0 +databricks-sql-connector==2.9.4 +datadog==0.48.0 db-dtypes==1.2.0 -debugpy==1.8.0 +debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.15.1 +deltalake==0.16.0 dill==0.3.1.1 distlib==0.3.8 distro==1.9.0 -dnspython==2.4.2 +dnspython==2.6.1 docker==7.0.0 docopt==0.6.2 docutils==0.20.1 -duckdb==0.9.2 -ecdsa==0.18.0 +duckdb==0.10.0 editables==0.5 -elastic-transport==8.11.0 -elasticsearch==8.11.1 +elastic-transport==8.12.0 +elasticsearch==8.12.1 email-validator==1.3.1 entrypoints==0.4 +enum34==1.1.10 eralchemy2==1.3.8 et-xmlfile==1.1.0 -eventlet==0.34.3 +eventlet==0.35.2 exceptiongroup==1.2.0 execnet==2.0.2 executing==2.0.1 -facebook-business==18.0.5 -fastavro==1.9.3 +facebook_business==19.0.0 +fastavro==1.9.4 fasteners==0.19 fastjsonschema==2.19.1 filelock==3.13.1 flower==2.0.1 frozenlist==1.4.1 -fsspec==2023.12.2 -future==0.18.3 +fsspec==2024.2.0 +future==1.0.0 gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==7.0.0 -gcloud-aio-storage==9.0.0 -gcsfs==2023.12.2.post1 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.2.0 +gcsfs==2024.2.0 geomet==0.2.1.post1 -gevent==23.9.1 +gevent==24.2.1 gitdb==4.0.11 -google-ads==22.1.0 -google-analytics-admin==0.22.2 -google-api-core==2.15.0 -google-api-python-client==2.113.0 +google-ads==23.1.0 +google-analytics-admin==0.22.7 +google-api-core==2.17.1 +google-api-python-client==2.122.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 -google-auth==2.26.2 -google-cloud-aiplatform==1.39.0 -google-cloud-appengine-logging==1.4.0 +google-auth==2.28.2 +google-cloud-aiplatform==1.43.0 +google-cloud-appengine-logging==1.4.3 google-cloud-audit-log==0.2.5 -google-cloud-automl==2.12.0 -google-cloud-batch==0.17.7 -google-cloud-bigquery-datatransfer==3.13.0 -google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.16.0 -google-cloud-bigtable==2.22.0 -google-cloud-build==3.22.0 -google-cloud-compute==1.15.0 -google-cloud-container==2.37.0 +google-cloud-automl==2.13.3 +google-cloud-batch==0.17.14 +google-cloud-bigquery-datatransfer==3.15.1 +google-cloud-bigquery==3.19.0 +google-cloud-bigtable==2.23.0 +google-cloud-build==3.23.3 +google-cloud-compute==1.18.0 +google-cloud-container==2.43.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.17.2 -google-cloud-dataflow-client==0.8.6 -google-cloud-dataform==0.5.5 -google-cloud-dataplex==1.11.0 -google-cloud-dataproc-metastore==1.14.0 -google-cloud-dataproc==5.8.0 -google-cloud-dlp==3.14.0 -google-cloud-kms==2.20.0 -google-cloud-language==2.12.0 +google-cloud-datacatalog==3.18.3 +google-cloud-dataflow-client==0.8.10 +google-cloud-dataform==0.5.9 +google-cloud-dataplex==1.12.3 +google-cloud-dataproc-metastore==1.15.3 +google-cloud-dataproc==5.9.3 +google-cloud-dlp==3.16.0 +google-cloud-kms==2.21.3 +google-cloud-language==2.13.3 google-cloud-logging==3.9.0 -google-cloud-memcache==1.8.0 -google-cloud-monitoring==2.18.0 -google-cloud-orchestration-airflow==1.10.0 -google-cloud-os-login==2.13.0 -google-cloud-pubsub==2.19.0 -google-cloud-redis==2.14.0 -google-cloud-resource-manager==1.11.0 -google-cloud-run==0.10.1 -google-cloud-secret-manager==2.17.0 -google-cloud-spanner==3.41.0 -google-cloud-speech==2.23.0 -google-cloud-storage-transfer==1.10.0 -google-cloud-storage==2.14.0 -google-cloud-tasks==2.15.0 -google-cloud-texttospeech==2.15.1 -google-cloud-translate==3.14.0 -google-cloud-videointelligence==2.12.0 -google-cloud-vision==3.5.0 -google-cloud-workflows==1.13.0 +google-cloud-memcache==1.9.3 +google-cloud-monitoring==2.19.3 +google-cloud-orchestration-airflow==1.12.1 +google-cloud-os-login==2.14.3 +google-cloud-pubsub==2.20.1 +google-cloud-redis==2.15.3 +google-cloud-resource-manager==1.12.3 +google-cloud-run==0.10.5 +google-cloud-secret-manager==2.18.3 +google-cloud-spanner==3.43.0 +google-cloud-speech==2.25.1 +google-cloud-storage-transfer==1.11.3 +google-cloud-storage==2.15.0 +google-cloud-tasks==2.16.3 +google-cloud-texttospeech==2.16.3 +google-cloud-translate==3.15.3 +google-cloud-videointelligence==2.13.3 +google-cloud-vision==3.7.2 +google-cloud-workflows==1.14.3 google-crc32c==1.5.0 google-re2==1.1 google-resumable-media==2.7.0 -googleapis-common-protos==1.62.0 +googleapis-common-protos==1.63.0 graphql-core==3.2.3 graphviz==0.20.1 greenlet==3.0.3 grpc-google-iam-v1==0.13.0 +grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-status==1.60.0 -grpcio==1.60.0 +grpcio-status==1.62.1 +grpcio==1.62.1 gssapi==1.8.3 gunicorn==21.2.0 h11==0.14.0 -hatch==1.9.1 -hatchling==1.21.0 +hatch==1.9.4 +hatchling==1.21.1 hdfs==2.7.3 hmsclient==0.1.1 httpcore==0.16.3 @@ -376,114 +379,115 @@ humanize==4.9.0 hvac==2.1.0 hyperlink==21.0.0 icdiff==2.0.7 -identify==2.5.33 +identify==2.5.35 idna==3.6 ijson==3.2.3 imagesize==1.4.1 importlib-metadata==6.11.0 -importlib-resources==6.1.1 +importlib_resources==6.1.3 impyla==0.19.0 incremental==22.10.0 inflection==0.5.1 -influxdb-client==1.39.0 +influxdb-client==1.41.0 iniconfig==2.0.0 ipdb==0.13.13 -ipykernel==6.28.0 +ipykernel==6.29.3 ipython==8.18.1 isodate==0.6.1 itsdangerous==2.1.2 -jaraco.classes==3.3.0 +jaraco.classes==3.3.1 jedi==0.19.1 jeepney==0.8.0 jmespath==0.10.0 +joserfc==0.9.0 jschema-to-python==1.2.3 json-merge-patch==0.2 jsondiff==2.0.0 jsonpatch==1.33 jsonpath-ng==1.6.1 -jsonpickle==3.0.2 +jsonpickle==3.0.3 jsonpointer==2.4 jsonschema-path==0.3.2 jsonschema-specifications==2023.12.1 -jsonschema==4.20.0 +jsonschema==4.21.1 junit-xml==1.9 -jupyter_client==8.6.0 -jupyter_core==5.7.1 -keyring==24.3.0 +jupyter_client==8.6.1 +jupyter_core==5.7.2 +keyring==24.3.1 kombu==5.3.5 krb5==0.5.1 -kubernetes-asyncio==24.2.3 -kubernetes==23.6.0 +kubernetes==29.0.0 +kubernetes_asyncio==29.0.0 kylinpy==2.8.4 lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.7.0 -linkify-it-py==2.0.2 +limits==3.10.0 +linkify-it-py==2.0.3 lockfile==0.12.2 loguru==0.7.2 -looker-sdk==23.20.1 +looker-sdk==24.2.0 lxml==5.1.0 lz4==4.3.3 markdown-it-py==3.0.0 -marshmallow-oneofschema==3.0.1 +marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 -marshmallow==3.20.2 +marshmallow==3.21.1 matplotlib-inline==0.1.6 mdit-py-plugins==0.4.0 mdurl==0.1.2 mmhash3==3.0.1 mongomock==4.1.2 more-itertools==10.2.0 -moto==4.2.13 +moto==5.0.3 mpmath==1.3.0 msal-extensions==1.1.0 -msal==1.26.0 +msal==1.27.0 msrest==0.7.1 msrestazure==0.6.4 -multi_key_dict==2.0.3 -multidict==6.0.4 +multi-key-dict==2.0.3 +multidict==6.0.5 mypy-boto3-appflow==1.34.0 -mypy-boto3-rds==1.34.6 +mypy-boto3-rds==1.34.58 mypy-boto3-redshift-data==1.34.0 mypy-boto3-s3==1.34.14 mypy-extensions==1.0.0 -mypy==1.2.0 -mysql-connector-python==8.0.29 -mysqlclient==2.2.1 -nbclient==0.9.0 -nbformat==5.9.2 -neo4j==5.16.0 -nest-asyncio==1.5.9 +mypy==1.8.0 +mysql-connector-python==8.3.0 +mysqlclient==2.2.4 +nbclient==0.9.1 +nbformat==5.10.2 +neo4j==5.18.0 +nest-asyncio==1.6.0 networkx==3.2.1 nh3==0.2.15 nodeenv==1.8.0 numpy==1.24.4 oauthlib==3.2.2 -objsize==0.6.1 -openai==1.7.2 +objsize==0.7.0 +openai==1.13.3 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 -openlineage-integration-common==1.7.0 -openlineage-python==1.7.0 -openlineage_sql==1.7.0 +openlineage-integration-common==1.9.1 +openlineage-python==1.9.1 +openlineage_sql==1.9.1 openpyxl==3.1.2 opensearch-py==2.4.2 -opentelemetry-api==1.22.0 -opentelemetry-exporter-otlp-proto-common==1.22.0 -opentelemetry-exporter-otlp-proto-grpc==1.22.0 -opentelemetry-exporter-otlp-proto-http==1.22.0 -opentelemetry-exporter-otlp==1.22.0 -opentelemetry-exporter-prometheus==0.43b0 -opentelemetry-proto==1.22.0 -opentelemetry-sdk==1.22.0 -opentelemetry-semantic-conventions==0.43b0 +opentelemetry-api==1.23.0 +opentelemetry-exporter-otlp-proto-common==1.23.0 +opentelemetry-exporter-otlp-proto-grpc==1.23.0 +opentelemetry-exporter-otlp-proto-http==1.23.0 +opentelemetry-exporter-otlp==1.23.0 +opentelemetry-exporter-prometheus==0.44b0 +opentelemetry-proto==1.23.0 +opentelemetry-sdk==1.23.0 +opentelemetry-semantic-conventions==0.44b0 opsgenie-sdk==2.1.5 -oracledb==2.0.1 +oracledb==2.1.0 ordered-set==4.1.0 -orjson==3.9.10 +orjson==3.9.15 oss2==2.18.4 packaging==23.2 -pandas-gbq==0.20.0 +pandas-gbq==0.22.0 pandas-stubs==2.0.2.230605 pandas==2.1.4 papermill==2.5.0 @@ -495,33 +499,33 @@ pbr==6.0.0 pdpyras==5.2.0 pendulum==3.0.0 pexpect==4.9.0 -pgvector==0.2.4 +pgvector==0.2.5 pinecone-client==2.2.4 -pinotdb==5.1.2 -pipdeptree==2.13.2 -pipx==1.4.2 -pkginfo==1.9.6 +pinotdb==5.1.3 +pipdeptree==2.16.1 +pipx==1.4.3 +pkginfo==1.10.0 platformdirs==3.11.0 -pluggy==1.3.0 +pluggy==1.4.0 ply==3.11 plyvel==1.5.1 portalocker==2.8.2 pprintpp==0.4.0 -pre-commit==3.6.0 +pre-commit==3.6.2 presto-python-client==0.8.4 prison==0.2.1 -prometheus-client==0.19.0 +prometheus_client==0.20.0 prompt-toolkit==3.0.43 proto-plus==1.23.0 -protobuf==4.25.2 -psutil==5.9.7 +protobuf==4.25.3 +psutil==5.9.8 psycopg2-binary==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 -py-partiql-parser==0.5.0 +py-partiql-parser==0.5.1 py4j==0.10.9.7 -pyOpenSSL==23.3.0 +pyOpenSSL==24.1.0 pyarrow-hotfix==0.6 pyarrow==14.0.2 pyasn1-modules==0.3.0 @@ -529,27 +533,26 @@ pyasn1==0.5.1 pycountry==23.12.11 pycparser==2.21 pycryptodome==3.20.0 -pydantic==2.5.3 -pydantic_core==2.14.6 +pydantic==2.6.4 +pydantic_core==2.16.3 pydata-google-auth==1.8.2 pydot==1.4.2 pydruid==0.6.6 pyenchant==3.2.2 pyexasol==0.25.2 pygraphviz==1.11 -pyiceberg==0.5.1 +pyiceberg==0.6.0 pyjsparser==2.7.1 pykerberos==1.2.4 -pymongo==4.6.1 +pymongo==4.6.2 pymssql==2.2.11 -pyodbc==5.0.1 -pyparsing==3.1.1 +pyodbc==5.1.0 +pyparsing==3.1.2 pypsrp==0.8.1 -pyspark==3.5.0 +pyspark==3.5.1 pyspnego==0.10.2 -pytest-asyncio==0.23.3 +pytest-asyncio==0.23.5.post1 pytest-cov==4.1.0 -pytest-httpx==0.21.3 pytest-icdiff==0.9 pytest-instafail==0.5.0 pytest-mock==3.12.0 @@ -557,45 +560,44 @@ pytest-rerunfailures==13.0 pytest-timeouts==1.2.1 pytest-xdist==3.5.0 pytest==7.4.4 -python-arango==7.9.0 +python-arango==7.9.1 python-daemon==3.0.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 python-http-client==3.3.7 python-jenkins==1.8.2 -python-jose==3.3.0 python-ldap==3.4.4 python-nvd3==0.15.0 -python-slugify==8.0.1 +python-slugify==8.0.4 python-telegram-bot==20.2 python3-saml==1.16.0 -pytz==2023.3.post1 +pytz==2024.1 pywinrm==0.4.3 pyzmq==25.1.2 reactivex==4.0.4 -readme-renderer==42.0 +readme_renderer==43.0 redis==4.6.0 -redshift-connector==2.0.918 -referencing==0.32.1 +redshift-connector==2.1.0 +referencing==0.33.0 regex==2023.12.25 -requests-file==1.5.1 +requests-file==2.0.0 requests-kerberos==0.14.0 requests-mock==1.11.0 requests-ntlm==1.2.0 -requests-oauthlib==1.3.1 +requests-oauthlib==1.4.0 requests-toolbelt==1.0.0 requests==2.31.0 -responses==0.24.1 -restructuredtext_lint==1.4.0 +responses==0.25.0 +restructuredtext-lint==1.4.0 rfc3339-validator==0.1.4 rfc3986==1.5.0 rich-argparse==1.4.0 -rich-click==1.7.3 -rich==13.7.0 -rpds-py==0.17.1 +rich-click==1.7.4 +rich==13.7.1 +rpds-py==0.18.0 rsa==4.9 -ruff==0.1.11 -s3fs==2023.12.2 +ruff==0.2.1 +s3fs==2024.2.0 s3transfer==0.8.2 sarif-om==1.0.4 scramp==1.4.4 @@ -603,18 +605,18 @@ scrapbook==0.5.0 semver==3.0.2 sendgrid==6.11.0 sentinels==1.0.0 -sentry-sdk==1.39.2 +sentry-sdk==1.41.0 setproctitle==1.3.3 -shapely==2.0.2 +shapely==2.0.3 shellingham==1.5.4 simple-salesforce==1.12.5 six==1.16.0 -slack_sdk==3.26.2 +slack_sdk==3.27.1 smbprotocol==1.12.0 smmap==5.0.1 -sniffio==1.3.0 +sniffio==1.3.1 snowballstemmer==2.2.0 -snowflake-connector-python==3.6.0 +snowflake-connector-python==3.7.1 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 @@ -636,12 +638,11 @@ sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-spelling==8.0.0 spython==0.3.13 -sqlalchemy-bigquery==1.9.0 +sqlalchemy-bigquery==1.10.0 +sqlalchemy-drill==1.1.4 sqlalchemy-redshift==0.8.14 sqlalchemy-spanner==1.6.2 -sqlalchemy_drill==1.1.4 sqlparse==0.4.4 -sshpubkeys==3.3.1 sshtunnel==0.4.0 stack-data==0.6.3 starkbank-ecdsa==2.2.0 @@ -655,64 +656,64 @@ termcolor==2.4.0 text-unidecode==1.3 thrift-sasl==0.4.3 thrift==0.16.0 -time-machine==2.13.0 +time-machine==2.14.0 tomli==2.0.1 tomli_w==1.0.0 -tomlkit==0.12.3 +tomlkit==0.12.4 tornado==6.4 towncrier==23.11.0 -tqdm==4.66.1 -traitlets==5.14.1 -trino==0.327.0 -trove-classifiers==2024.1.8 -twine==4.0.2 -types-Deprecated==1.2.9.20240106 -types-Markdown==3.5.0.20240106 +tqdm==4.66.2 +traitlets==5.14.2 +trino==0.328.0 +trove-classifiers==2024.3.3 +twine==5.0.0 +types-Deprecated==1.2.9.20240311 +types-Markdown==3.5.0.20240311 types-PyMySQL==1.1.0.1 -types-PyYAML==6.0.12.12 -types-aiofiles==23.2.0.20240106 +types-PyYAML==6.0.12.20240311 +types-aiofiles==23.2.0.20240311 types-certifi==2021.10.8.3 types-croniter==2.0.0.20240106 -types-docutils==0.20.0.20240106 -types-paramiko==3.4.0.20240106 -types-protobuf==4.24.0.20240106 -types-pyOpenSSL==23.3.0.20240106 -types-python-dateutil==2.8.19.20240106 -types-python-slugify==8.0.0.3 -types-pytz==2023.3.1.1 -types-redis==4.6.0.20240106 +types-docutils==0.20.0.20240311 +types-paramiko==3.4.0.20240311 +types-protobuf==4.24.0.20240311 +types-pyOpenSSL==24.0.0.20240311 +types-python-dateutil==2.8.19.20240311 +types-python-slugify==8.0.2.20240310 +types-pytz==2024.1.0.20240203 +types-redis==4.6.0.20240311 types-requests==2.31.0.6 -types-setuptools==69.0.0.20240115 +types-setuptools==69.1.0.20240310 types-tabulate==0.9.0.20240106 types-termcolor==1.1.6.2 -types-toml==0.10.8.7 +types-toml==0.10.8.20240310 types-urllib3==1.26.25.14 -typing_extensions==4.9.0 -tzdata==2023.4 +typing_extensions==4.10.0 +tzdata==2024.1 tzlocal==5.2 -uc-micro-py==1.0.2 +uc-micro-py==1.0.3 unicodecsv==0.14.1 universal-pathlib==0.1.4 uritemplate==4.1.1 urllib3==1.26.18 -userpath==1.9.1 +userpath==1.9.2 validators==0.22.0 vertica-python==1.3.8 vine==5.1.0 -virtualenv==20.25.0 -watchtower==3.0.1 +virtualenv==20.25.1 +watchtower==3.1.0 wcwidth==0.2.13 -weaviate-client==3.26.1 +weaviate-client==3.26.2 websocket-client==1.7.0 wrapt==1.16.0 xmlsec==1.3.13 xmltodict==0.13.0 -yamllint==1.33.0 -yandexcloud==0.253.0 +yamllint==1.35.1 +yandexcloud==0.264.0 yarl==1.9.4 zeep==4.2.1 -zenpy==2.0.45 +zenpy==2.0.47 zipp==3.17.0 zope.event==5.0 -zope.interface==6.1 +zope.interface==6.2 zstandard==0.22.0 From e84584122fb497a292f942748e52697d5a59982a Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 13:48:38 +0100 Subject: [PATCH 4/6] fix(airflow): markdown lints --- airflow/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/README.md b/airflow/README.md index 221841c9a..0efd37979 100644 --- a/airflow/README.md +++ b/airflow/README.md @@ -1,8 +1,8 @@ # Airflow constraints -To add a new version of airflow (or python) you can download the +To add a new version of Airflow (or python) you can download the applicable constraints file from upstream. -_The script is safe to run from outside of this directory. The file will be downloaded to the same folder that the +_The script is safe to run from outside of this directory. The file will be downloaded to the same folder that the script resides. This example will assume you are running from the repository root directory._ ```sh @@ -15,7 +15,7 @@ script resides. This example will assume you are running from the repository roo Example output: -``` +```output Downloading constraints file for Airflow 2.8.3 (Python 3.9) Successfully pulled new constraints file: constraints-2.8.3-python39.txt ``` From 1f4142d19935f10cd79537a1b40616a0e3e833b9 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 13:50:53 +0100 Subject: [PATCH 5/6] fix(airflow): shellcheck lints --- airflow/download_constraints.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/download_constraints.sh b/airflow/download_constraints.sh index 3eff91225..b3924d11c 100755 --- a/airflow/download_constraints.sh +++ b/airflow/download_constraints.sh @@ -8,10 +8,10 @@ FILENAME="constraints-${AIRFLOW_VERSION}-python${PYTHON_VERSION//.}.txt" # Find the directory name of the script so it can be run from outside DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd ${DIR} > /dev/null +pushd ${DIR} > /dev/null || exit 1 echo "Downloading constraints file for Airflow ${AIRFLOW_VERSION} (Python ${PYTHON_VERSION})" curl --fail -Ls "${URL}" -o "${FILENAME}" echo "Successfully pulled new constraints file: ${FILENAME}" -popd > /dev/null +popd > /dev/null || exit 1 From 3719622df39b70bae86c760073e6951219fd3372 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 13 Mar 2024 13:53:09 +0100 Subject: [PATCH 6/6] fix(airflow): shellcheck lints --- airflow/download_constraints.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/download_constraints.sh b/airflow/download_constraints.sh index b3924d11c..7cffee658 100755 --- a/airflow/download_constraints.sh +++ b/airflow/download_constraints.sh @@ -8,7 +8,7 @@ FILENAME="constraints-${AIRFLOW_VERSION}-python${PYTHON_VERSION//.}.txt" # Find the directory name of the script so it can be run from outside DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd ${DIR} > /dev/null || exit 1 +pushd "${DIR}" > /dev/null || exit 1 echo "Downloading constraints file for Airflow ${AIRFLOW_VERSION} (Python ${PYTHON_VERSION})" curl --fail -Ls "${URL}" -o "${FILENAME}"