diff --git a/CHANGELOG.md b/CHANGELOG.md index dda2e1655..8cdb6dc83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. - opa: Add `1.16.2` ([#1509]). - kafka: Add `3.9.2` and `4.2.1` ([#1483]). - trino: Add `481` ([#1518]). +- airflow: Add `3.2.1` ([#1519]). ### Changed @@ -51,6 +52,7 @@ All notable changes to this project will be documented in this file. [#1510]: https://github.com/stackabletech/docker-images/pull/1510 [#1512]: https://github.com/stackabletech/docker-images/pull/1512 [#1518]: https://github.com/stackabletech/docker-images/pull/1518 +[#1519]: https://github.com/stackabletech/docker-images/pull/1519 [#1520]: https://github.com/stackabletech/docker-images/pull/1520 ## [26.3.0] - 2026-03-16 diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 12cddcb02..a90813a20 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -164,6 +164,7 @@ else fi # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3 +# TODO: s3fs may already be installed via the s3fs extra and constraints. Check if this explicit install is still needed. uv pip install --no-cache-dir s3fs==${S3FS_VERSION} cyclonedx-bom==${CYCLONEDX_BOM_VERSION} # Needed for OIDC uv pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1 @@ -172,11 +173,7 @@ uv pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl # Create the SBOM for Airflow # Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete -if [ "$PRODUCT_VERSION" == "2.9.3" ] || [ "$PRODUCT_VERSION" == "3.0.6" ]; then - cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json -else - cyclonedx-py environment --spec-version 1.5 --output-file /tmp/sbom.json -fi +cyclonedx-py environment --spec-version 1.5 --output-file /tmp/sbom.json uv pip uninstall cyclonedx-bom # Break circular dependencies by removing the apache-airflow dependency from the providers diff --git a/airflow/boil-config.toml b/airflow/boil-config.toml index 7835a1352..a915c50f9 100644 --- a/airflow/boil-config.toml +++ b/airflow/boil-config.toml @@ -8,12 +8,15 @@ vector = "0.55.0" stackable-devel = "1.0.0" [versions."2.9.3".build-arguments] +# NOTE: Python version is kept stable unless the new Airflow version drops support for it. python-version = "3.9" -git-sync-version = "v4.4.1" +git-sync-version = "v4.6.0" +# Must match the version in the constraints file. +# TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints. s3fs-version = "2024.9.0" -cyclonedx-bom-version = "6.0.0" +cyclonedx-bom-version = "7.3.0" tini-version = "0.19.0" -uv-version = "0.7.8" +uv-version = "0.11.7" airflow-extras-other = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino" opa-auth-manager = "airflow-2" nodejs-version = "20" @@ -25,14 +28,19 @@ vector = "0.55.0" stackable-devel = "1.0.0" [versions."3.0.6".build-arguments] +# NOTE: Python version is kept stable unless the new Airflow version drops support for it. python-version = "3.12" -git-sync-version = "v4.4.1" +git-sync-version = "v4.6.0" +# Must match the version in the constraints file. +# TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints. s3fs-version = "2024.9.0" -cyclonedx-bom-version = "6.0.0" +cyclonedx-bom-version = "7.3.0" tini-version = "0.19.0" -uv-version = "0.7.8" +uv-version = "0.11.7" -# Airflow extras are defined in separate lists to make them easier to check against the links below. The lists will be concatenated and duplicates removed in the dockerfile. +# Airflow extras are defined in separate lists to make them easier to check against the links below. +# The lists will be concatenated and duplicates removed in the dockerfile. +# TODO: Add policy for when to add extras. # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#core-airflow-extras airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd" @@ -54,39 +62,83 @@ airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp, opa-auth-manager = "airflow-3" nodejs-version = "20" -# Supported +# Deprecated [versions."3.1.6".local-images] "shared/statsd-exporter" = "0.28.0" vector = "0.55.0" stackable-devel = "1.0.0" [versions."3.1.6".build-arguments] -# NOTE (@Techassi): Maybe this version can be bumped +# NOTE: Python version is kept stable unless the new Airflow version drops support for it. python-version = "3.12" -git-sync-version = "v4.5.1" +git-sync-version = "v4.6.0" +# Must match the version in the constraints file. +# TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints. s3fs-version = "2026.1.0" -cyclonedx-bom-version = "7.2.1" +cyclonedx-bom-version = "7.3.0" +tini-version = "0.19.0" +uv-version = "0.11.7" + +# Airflow extras are defined in separate lists to make them easier to check against the links below. +# The lists will be concatenated and duplicates removed in the dockerfile. +# TODO: Add policy for when to add extras. +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#core-airflow-extras +airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd" + +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#meta-airflow-package-extras +airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv" + +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#apache-software-extras +airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot" + +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#external-services-extras +airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk" + +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#locally-installed-software-extras +airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino" + +# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#other-extras +airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh" + +opa-auth-manager = "airflow-3" +nodejs-version = "22" + +# Supported +[versions."3.2.1".local-images] +"shared/statsd-exporter" = "0.28.0" +vector = "0.55.0" +stackable-devel = "1.0.0" + +[versions."3.2.1".build-arguments] +# NOTE: Python version is kept stable unless the new Airflow version drops support for it. +python-version = "3.12" +git-sync-version = "v4.6.0" +# Must match the version in the constraints file. +# TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints. +s3fs-version = "2026.3.0" +cyclonedx-bom-version = "7.3.0" tini-version = "0.19.0" -# NOTE (@Techassi): Maybe this version can be bumped -uv-version = "0.7.22" +uv-version = "0.11.7" -# Airflow extras are defined in separate lists to make them easier to check against the links below. The lists will be concatenated and duplicates removed in the dockerfile. -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#core-airflow-extras +# Airflow extras are defined in separate lists to make them easier to check against the links below. +# The lists will be concatenated and duplicates removed in the dockerfile. +# TODO: Add policy for when to add extras. +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#core-airflow-extras airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd" -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#meta-airflow-package-extras +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#meta-airflow-package-extras airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv" -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#apache-software-extras +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#apache-software-extras airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot" -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#external-services-extras +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#external-services-extras airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk" -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#locally-installed-software-extras +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#locally-installed-software-extras airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino" -# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#other-extras +# See https://airflow.apache.org/docs/apache-airflow/3.2.1/extra-packages-ref.html#other-extras airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh" opa-auth-manager = "airflow-3" diff --git a/airflow/stackable/constraints/3.2.1/build-constraints-python3.12.txt b/airflow/stackable/constraints/3.2.1/build-constraints-python3.12.txt new file mode 100644 index 000000000..c2761bf04 --- /dev/null +++ b/airflow/stackable/constraints/3.2.1/build-constraints-python3.12.txt @@ -0,0 +1 @@ +Cython>=3.1.0 diff --git a/airflow/stackable/constraints/3.2.1/constraints-python3.12.txt b/airflow/stackable/constraints/3.2.1/constraints-python3.12.txt new file mode 100644 index 000000000..caf107584 --- /dev/null +++ b/airflow/stackable/constraints/3.2.1/constraints-python3.12.txt @@ -0,0 +1,710 @@ + +# +# This constraints file was automatically generated on 2026-04-21T00:39:47.116640 +# via `uv pip install --resolution highest` for the "v3-2-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# This constraints file is meant to be used only in the "apache-airflow" installation command and not +# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow +# installation step is reproducible. Subsequent pip commands may install packages that would have +# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip +# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" +# in the list of install targets to prevent Airflow accidental upgrade or downgrade. +# +# Typical installation process of airflow for Python 3.12 is (with random selection of extras and custom +# dependencies added), usually consists of two steps: +# +# 1. Reproducible installation of airflow with selected providers (note constraints are used): +# +# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ +# --constraint \ +# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.12.txt" +# +# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not +# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. +# +# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" +# +APScheduler==3.11.2 +Authlib==1.6.11 +Deprecated==1.3.1 +Events==0.5 +Flask-JWT-Extended==4.7.1 +Flask-Limiter==3.12 +Flask-Login==0.6.3 +Flask-SQLAlchemy==3.1.1 +Flask-Session==0.8.0 +Flask-WTF==1.2.2 +Flask==3.1.3 +GitPython==3.1.46 +JayDeBeApi==1.2.3 +Jinja2==3.1.6 +Mako==1.3.11 +Markdown==3.10.2 +MarkupSafe==3.0.3 +PyAthena==3.30.1 +PyGithub==2.9.1 +PyHive==0.7.0 +PyJWT==2.12.1 +PyMySQL==1.1.2 +PyNaCl==1.6.2 +PyYAML==6.0.3 +Pygments==2.20.0 +SQLAlchemy-Utils==0.42.1 +SQLAlchemy==2.0.49 +SecretStorage==3.5.0 +WTForms==3.2.1 +Werkzeug==3.1.8 +a2wsgi==1.10.10 +adal==1.2.7 +adbc-driver-manager==1.11.0 +adbc-driver-postgresql==1.11.0 +adbc-driver-sqlite==1.11.0 +adlfs==2026.4.0 +aenum==3.1.17 +aiobotocore==3.4.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp-cors==0.8.1 +aiohttp==3.13.5 +aioitertools==0.13.0 +aiomysql==0.3.2 +aiosignal==1.4.0 +aiosmtplib==5.1.0 +aiosqlite==0.21.0 +airbyte-api==0.53.0 +alembic==1.18.4 +alibabacloud-adb20211201==3.7.1 +alibabacloud-credentials-api==1.0.0 +alibabacloud-credentials==1.0.8 +alibabacloud-oss-v2==1.2.5 +alibabacloud-tea-openapi==0.4.4 +alibabacloud-tea-util==0.3.14 +alibabacloud-tea==0.4.3 +alibabacloud_gateway_spi==0.0.3 +amqp==5.3.1 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anyio==4.13.0 +apache-airflow-providers-airbyte==5.4.1 +apache-airflow-providers-alibaba==3.3.7 +apache-airflow-providers-amazon==9.25.0 +apache-airflow-providers-apache-cassandra==3.9.4 +apache-airflow-providers-apache-drill==3.3.2 +apache-airflow-providers-apache-druid==4.5.2 +apache-airflow-providers-apache-flink==1.8.4 +apache-airflow-providers-apache-hdfs==4.11.5 +apache-airflow-providers-apache-hive==9.4.2 +apache-airflow-providers-apache-iceberg==2.0.2 +apache-airflow-providers-apache-impala==1.9.2 +apache-airflow-providers-apache-kafka==1.13.2 +apache-airflow-providers-apache-kylin==3.10.4 +apache-airflow-providers-apache-livy==4.5.5 +apache-airflow-providers-apache-pig==4.8.4 +apache-airflow-providers-apache-pinot==4.10.2 +apache-airflow-providers-apache-spark==6.0.1 +apache-airflow-providers-apache-tinkerpop==1.1.3 +apache-airflow-providers-apprise==2.3.3 +apache-airflow-providers-arangodb==2.9.4 +apache-airflow-providers-asana==2.11.3 +apache-airflow-providers-atlassian-jira==3.3.3 +apache-airflow-providers-celery==3.18.0 +apache-airflow-providers-cloudant==4.3.4 +apache-airflow-providers-cncf-kubernetes==10.16.0 +apache-airflow-providers-cohere==1.6.5 +apache-airflow-providers-common-ai==0.1.0 +apache-airflow-providers-common-compat==1.14.3 +apache-airflow-providers-common-io==1.7.2 +apache-airflow-providers-common-messaging==2.0.3 +apache-airflow-providers-common-sql==1.34.0 +apache-airflow-providers-databricks==7.12.1 +apache-airflow-providers-datadog==3.10.4 +apache-airflow-providers-dbt-cloud==4.8.1 +apache-airflow-providers-dingding==3.9.4 +apache-airflow-providers-discord==3.12.2 +apache-airflow-providers-docker==4.5.5 +apache-airflow-providers-edge3==3.4.0 +apache-airflow-providers-elasticsearch==6.5.2 +apache-airflow-providers-exasol==4.10.2 +apache-airflow-providers-fab==3.6.1 +apache-airflow-providers-facebook==3.9.4 +apache-airflow-providers-ftp==3.14.3 +apache-airflow-providers-git==0.3.1 +apache-airflow-providers-github==2.11.2 +apache-airflow-providers-google==21.1.0 +apache-airflow-providers-grpc==3.9.4 +apache-airflow-providers-hashicorp==4.5.2 +apache-airflow-providers-http==6.0.2 +apache-airflow-providers-imap==3.11.2 +apache-airflow-providers-influxdb==2.10.4 +apache-airflow-providers-informatica==0.1.3 +apache-airflow-providers-jdbc==5.4.3 +apache-airflow-providers-jenkins==4.2.5 +apache-airflow-providers-keycloak==0.7.1 +apache-airflow-providers-microsoft-azure==13.1.1 +apache-airflow-providers-microsoft-mssql==4.5.2 +apache-airflow-providers-microsoft-psrp==3.2.5 +apache-airflow-providers-microsoft-winrm==3.14.2 +apache-airflow-providers-mongo==5.3.5 +apache-airflow-providers-mysql==6.5.2 +apache-airflow-providers-neo4j==3.11.5 +apache-airflow-providers-odbc==4.12.2 +apache-airflow-providers-openai==1.7.4 +apache-airflow-providers-openfaas==3.9.4 +apache-airflow-providers-openlineage==2.14.0 +apache-airflow-providers-opensearch==1.9.0 +apache-airflow-providers-opsgenie==5.10.3 +apache-airflow-providers-oracle==4.5.3 +apache-airflow-providers-pagerduty==5.2.5 +apache-airflow-providers-papermill==3.12.3 +apache-airflow-providers-pgvector==1.7.1 +apache-airflow-providers-pinecone==2.4.4 +apache-airflow-providers-postgres==6.6.3 +apache-airflow-providers-presto==5.11.2 +apache-airflow-providers-qdrant==1.5.5 +apache-airflow-providers-redis==4.4.4 +apache-airflow-providers-salesforce==5.14.0 +apache-airflow-providers-samba==4.12.5 +apache-airflow-providers-segment==3.9.4 +apache-airflow-providers-sendgrid==4.2.2 +apache-airflow-providers-sftp==5.7.3 +apache-airflow-providers-singularity==3.9.3 +apache-airflow-providers-slack==9.10.0 +apache-airflow-providers-smtp==2.4.5 +apache-airflow-providers-snowflake==6.12.1 +apache-airflow-providers-sqlite==4.3.2 +apache-airflow-providers-ssh==5.0.0 +apache-airflow-providers-standard==1.12.3 +apache-airflow-providers-tableau==5.4.0 +apache-airflow-providers-telegram==4.9.4 +apache-airflow-providers-teradata==3.5.2 +apache-airflow-providers-trino==6.5.2 +apache-airflow-providers-vertica==4.3.2 +apache-airflow-providers-weaviate==3.3.3 +apache-airflow-providers-yandex==4.4.2 +apache-airflow-providers-ydb==2.5.2 +apache-airflow-providers-zendesk==4.11.3 +apispec==6.10.0 +apprise==1.9.9 +argcomplete==3.6.3 +asana==5.2.4 +asgiref==3.11.1 +asn1crypto==1.5.1 +asttokens==3.0.1 +async-timeout==4.0.3 +asyncpg==0.31.0 +asyncssh==2.22.0 +atlasclient==1.0.0 +atlassian-python-api==4.0.7 +attrs==26.1.0 +azure-batch==14.2.0 +azure-common==1.1.28 +azure-core==1.39.0 +azure-cosmos==4.15.0 +azure-datalake-store==0.0.53 +azure-identity==1.25.3 +azure-keyvault-secrets==4.10.0 +azure-kusto-data==6.0.3 +azure-mgmt-compute==37.2.0 +azure-mgmt-containerinstance==10.1.0 +azure-mgmt-containerregistry==15.0.0 +azure-mgmt-core==1.6.0 +azure-mgmt-cosmosdb==9.9.0 +azure-mgmt-datafactory==9.3.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==25.0.0 +azure-mgmt-storage==24.0.1 +azure-nspkg==3.0.2 +azure-servicebus==7.14.3 +azure-storage-blob==12.28.0 +azure-storage-file-datalake==12.23.0 +azure-storage-file-share==12.24.0 +azure-synapse-artifacts==0.22.0 +azure-synapse-spark==0.7.0 +babel==2.18.0 +backoff==2.2.1 +bcrypt==5.0.0 +beautifulsoup4==4.14.3 +billiard==4.2.4 +bitarray==3.8.1 +black==26.3.1 +bleach==6.3.0 +blinker==1.9.0 +boto3==1.42.84 +botocore==1.42.84 +cachelib==0.13.0 +cachetools==6.2.6 +cadwyn==6.2.0 +cassandra-driver==3.30.0 +cattrs==26.1.0 +celery==5.6.3 +certifi==2026.2.25 +cffi==2.0.0 +chardet==6.0.0.post1 +charset-normalizer==3.4.7 +ciso8601==2.3.3 +click-didyoumean==0.3.1 +click-plugins==1.1.1.2 +click-repl==0.3.0 +click==8.3.2 +cloudpickle==3.1.1 +cohere==5.21.1 +colorama==0.4.6 +colorful==0.5.8 +colorlog==6.10.1 +comm==0.2.3 +confluent-kafka==2.14.0 +crcmod-plus==2.3.1 +cron_descriptor==2.0.8 +croniter==6.2.2 +cryptography==46.0.7 +curlify==3.0.0 +darabonba-core==1.0.5 +databricks-sql-connector==4.2.5 +dataclasses-json==0.6.7 +datadog==0.52.1 +db-dtypes==1.5.1 +debugpy==1.8.20 +decorator==5.2.1 +defusedxml==0.7.1 +deprecation==2.1.0 +dill==0.4.1 +distlib==0.4.0 +distro==1.9.0 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.18.0 +durationpy==0.10 +ecdsa==0.19.2 +elastic-transport==9.2.1 +elasticsearch==9.3.0 +email-validator==2.3.0 +entrypoints==0.4 +et_xmlfile==2.0.0 +eventlet==0.41.0 +executing==2.2.1 +facebook_business==25.0.1 +fastapi-cli==0.0.24 +fastapi==0.136.0 +fastavro==1.12.1 +fastjsonschema==2.21.2 +fastuuid==0.14.0 +filelock==3.28.0 +flask-appbuilder==5.2.0 +flask-babel==4.0.0 +flower==2.0.1 +frozenlist==1.8.0 +fsspec==2026.3.0 +future==1.0.0 +gcloud-aio-auth==5.4.4 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.6.4 +gcsfs==2026.3.0 +genai-prices==0.0.56 +geomet==1.1.0 +gevent==26.4.0 +gitdb==4.0.12 +google-ads==30.0.0 +google-analytics-admin==0.28.0 +google-api-core==2.30.3 +google-api-python-client==2.194.0 +google-auth-httplib2==0.3.1 +google-auth-oauthlib==1.3.1 +google-auth==2.49.2 +google-cloud-aiplatform==1.148.0 +google-cloud-alloydb==0.9.0 +google-cloud-appengine-logging==1.9.0 +google-cloud-audit-log==0.5.0 +google-cloud-automl==2.19.0 +google-cloud-batch==0.21.0 +google-cloud-bigquery-datatransfer==3.22.0 +google-cloud-bigquery-storage==2.37.0 +google-cloud-bigquery==3.41.0 +google-cloud-bigtable==2.36.0 +google-cloud-build==3.36.0 +google-cloud-compute==1.47.0 +google-cloud-container==2.64.0 +google-cloud-core==2.5.1 +google-cloud-datacatalog==3.30.0 +google-cloud-dataflow-client==0.13.0 +google-cloud-dataform==0.10.0 +google-cloud-dataplex==2.18.0 +google-cloud-dataproc-metastore==1.22.0 +google-cloud-dataproc==5.27.0 +google-cloud-dlp==3.36.0 +google-cloud-kms==3.12.0 +google-cloud-language==2.20.0 +google-cloud-logging==3.15.0 +google-cloud-managedkafka==0.4.0 +google-cloud-memcache==1.15.0 +google-cloud-monitoring==2.30.0 +google-cloud-orchestration-airflow==1.20.0 +google-cloud-os-login==2.20.0 +google-cloud-pubsub==2.37.0 +google-cloud-redis==2.21.0 +google-cloud-resource-manager==1.17.0 +google-cloud-run==0.16.0 +google-cloud-secret-manager==2.27.0 +google-cloud-spanner==3.65.0 +google-cloud-speech==2.38.0 +google-cloud-storage-control==1.11.0 +google-cloud-storage-transfer==1.20.0 +google-cloud-storage==3.10.1 +google-cloud-tasks==2.22.0 +google-cloud-texttospeech==2.36.0 +google-cloud-translate==3.26.0 +google-cloud-videointelligence==2.19.0 +google-cloud-vision==3.13.0 +google-cloud-workflows==1.21.0 +google-crc32c==1.8.0 +google-genai==1.73.1 +google-resumable-media==2.8.2 +googleapis-common-protos==1.74.0 +graphviz==0.21 +greenback==1.3.0 +greenlet==3.4.0 +gremlinpython==3.8.1 +griffelib==2.0.2 +grpc-google-iam-v1==0.14.4 +grpc-interceptor==0.15.4 +grpcio-gcp==0.2.2 +grpcio-health-checking==1.80.0 +grpcio-status==1.80.0 +grpcio-tools==1.80.0 +grpcio==1.80.0 +gssapi==1.11.1 +gunicorn==25.3.0 +h11==0.16.0 +h2==4.3.0 +hdfs==2.7.3 +hf-xet==1.4.3 +hmsclient==0.1.1 +hpack==4.1.0 +httpcore==1.0.9 +httplib2==0.31.2 +httptools==0.7.1 +httpx==0.28.1 +huggingface_hub==1.11.0 +humanize==4.15.0 +hvac==2.4.0 +hyperframe==6.1.0 +ibm-cloud-sdk-core==3.24.4 +ibmcloudant==0.11.5 +idna==3.11 +ijson==3.4.0.post0 +immutabledict==4.3.1 +importlib_metadata==8.7.1 +importlib_resources==7.1.0 +impyla==0.22.0 +inflection==0.5.1 +influxdb-client==1.50.0 +ipykernel==7.2.0 +ipython==9.12.0 +ipython_pygments_lexers==1.1.1 +isodate==0.7.2 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.1.2 +jaraco.functools==4.4.0 +jedi==0.19.2 +jeepney==0.9.0 +jiter==0.14.0 +jmespath==1.1.0 +joblib==1.5.3 +jpype1==1.7.0 +jsonpath-ng==1.8.0 +jsonpath-python==1.1.5 +jsonschema-specifications==2025.9.1 +jsonschema==4.26.0 +jupyter_client==8.8.0 +jupyter_core==5.9.1 +jupyterlab_pygments==0.3.0 +jwcrypto==1.5.7 +keyring==25.7.0 +kombu==5.6.2 +krb5==0.9.0 +kubernetes==35.0.0 +kubernetes_asyncio==35.0.1 +kylinpy==2.8.4 +lazy-object-proxy==1.12.0 +libcst==1.8.6 +limits==5.8.0 +linkify-it-py==2.1.0 +litellm==1.82.6 +lockfile==0.12.2 +logfire-api==4.32.1 +looker_sdk==26.6.1 +lxml==6.0.2 +lz4==4.4.5 +markdown-it-py==4.0.0 +marshmallow-sqlalchemy==1.5.0 +marshmallow==3.26.2 +matplotlib-inline==0.2.1 +mdit-py-plugins==0.5.0 +mdurl==0.1.2 +memray==1.19.3 +mergedeep==1.3.4 +methodtools==0.4.7 +microsoft-kiota-abstractions==1.10.1 +microsoft-kiota-authentication-azure==1.10.1 +microsoft-kiota-http==1.10.1 +microsoft-kiota-serialization-json==1.10.1 +microsoft-kiota-serialization-text==1.10.1 +mistune==3.2.0 +mmh3==5.2.1 +more-itertools==11.0.2 +msal-extensions==1.3.1 +msal==1.36.0 +msgpack==1.1.2 +msgraph-core==1.3.8 +msgraphfs==0.4 +msgspec==0.21.1 +msrest==0.7.1 +msrestazure==0.6.4.post1 +multi_key_dict==2.0.3 +multidict==6.7.1 +mypy_extensions==1.1.0 +mysql-connector-python==9.6.0 +mysqlclient==2.2.8 +natsort==8.4.0 +nbclient==0.10.4 +nbconvert==7.17.1 +nbformat==5.10.4 +neo4j==6.1.0 +nest-asyncio==1.6.0 +numpy==2.4.4 +oauthlib==3.3.1 +openai==2.32.0 +opencensus-context==0.1.3 +opencensus==0.11.4 +openlineage-integration-common==1.46.0 +openlineage-python==1.46.0 +openlineage_sql==1.46.0 +openpyxl==3.1.5 +opensearch-protobufs==0.19.0 +opensearch-py==3.1.0 +opentelemetry-api==1.41.0 +opentelemetry-exporter-otlp-proto-common==1.41.0 +opentelemetry-exporter-otlp-proto-grpc==1.41.0 +opentelemetry-exporter-otlp-proto-http==1.41.0 +opentelemetry-exporter-otlp==1.41.0 +opentelemetry-exporter-prometheus==0.62b0 +opentelemetry-proto==1.41.0 +opentelemetry-resourcedetector-gcp==1.11.0a0 +opentelemetry-sdk==1.41.0 +opentelemetry-semantic-conventions==0.62b0 +opsgenie-sdk==2.1.5 +oracledb==3.4.2 +ordered-set==4.1.0 +orjson==3.11.8 +outcome==1.3.0.post0 +packaging==26.1 +pagerduty==6.2.1 +pandas-gbq==0.35.0 +pandas-stubs==3.0.0.260204 +pandas==2.3.3 +pandocfilters==1.5.1 +papermill==2.7.0 +paramiko==3.5.1 +parso==0.8.6 +pathlib_abc==0.5.2 +pathspec==1.0.4 +pbr==7.0.3 +pendulum==3.2.0 +pexpect==4.9.0 +pgvector==0.4.2 +pinecone-plugin-assistant==3.0.3 +pinecone-plugin-interface==0.0.7 +pinecone==8.1.2 +pinotdb==9.1.1 +platformdirs==4.9.6 +pluggy==1.6.0 +polars-runtime-32==1.39.3 +polars==1.39.3 +portalocker==3.2.0 +presto-python-client==0.8.4 +prison==0.2.1 +prometheus_client==0.25.0 +prompt_toolkit==3.0.52 +propcache==0.4.1 +proto-plus==1.27.2 +protobuf==6.33.6 +psutil==7.2.2 +psycopg2-binary==2.9.11 +ptyprocess==0.7.0 +pure-sasl==0.6.2 +pure_eval==0.2.3 +py-spy==0.4.1 +pyOpenSSL==26.0.0 +pyarrow==23.0.1 +pyasn1==0.6.3 +pyasn1_modules==0.4.2 +pybreaker==1.4.1 +pycountry==26.2.16 +pycparser==3.0 +pycryptodome==3.23.0 +pydantic-ai-slim==1.84.0 +pydantic-extra-types==2.11.1 +pydantic-graph==1.84.0 +pydantic-settings==2.13.1 +pydantic==2.13.1 +pydantic_core==2.46.1 +pydata-google-auth==1.9.1 +pydruid==0.6.9 +pyexasol==1.3.0 +pygtrie==2.5.0 +pyiceberg==0.11.1 +pykerberos==1.2.4 +pymongo==4.16.0 +pymssql==2.3.13 +pyodbc==5.3.0 +pyodps==0.12.6 +pyparsing==3.3.2 +pypsrp==0.9.1 +pyroaring==1.0.4 +pyspark-client==4.1.1 +pyspnego==0.12.1 +python-arango==8.3.2 +python-daemon==3.1.2 +python-dateutil==2.9.0.post0 +python-discovery==1.2.2 +python-dotenv==1.2.2 +python-http-client==3.3.7 +python-jenkins==1.8.3 +python-keycloak==7.1.1 +python-ldap==3.4.5 +python-multipart==0.0.26 +python-slugify==8.0.4 +python-telegram-bot==22.7 +python3-saml==1.16.0 +pytokens==0.4.1 +pytz==2026.1.post1 +pywinrm==0.5.0 +pyzmq==27.1.0 +qdrant-client==1.17.1 +ray==2.55.0 +reactivex==4.1.0 +redis==6.4.0 +redshift_connector==2.1.13 +referencing==0.37.0 +regex==2026.4.4 +requests-file==3.0.1 +requests-kerberos==0.15.0 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requests==2.33.1 +requests_ntlm==1.3.0 +retryhttp==1.4.0 +rich-argparse==1.7.2 +rich-toolkit==0.19.7 +rich==13.9.4 +rpds-py==0.30.0 +rsa==4.9.1 +ruamel.yaml==0.19.1 +s3fs==2026.3.0 +s3transfer==0.16.0 +sagemaker_studio==1.0.26 +scikit-learn==1.8.0 +scipy==1.17.1 +scramp==1.4.8 +scrapbook==0.5.0 +segment-analytics-python==2.3.6 +sendgrid==6.12.2 +sentry-sdk==2.58.0 +setproctitle==1.3.7 +setuptools==82.0.1 +shellingham==1.5.4 +simple-salesforce==1.12.9 +six==1.17.0 +slack_sdk==3.41.0 +smart_open==7.6.0 +smbprotocol==1.16.1 +smmap==5.0.3 +sniffio==1.3.1 +snowflake-connector-python==4.4.0 +snowflake-snowpark-python==1.49.0 +snowflake-sqlalchemy==1.9.0 +sortedcontainers==2.4.0 +soupsieve==2.8.3 +spython==0.3.14 +sqlalchemy-bigquery==1.16.0 +sqlalchemy-spanner==1.17.3 +sqlalchemy_drill==1.1.10 +sqlparse==0.5.5 +stack-data==0.6.3 +starlette==0.52.1 +statsd==4.0.1 +std-uritemplate==2.0.8 +strictyaml==1.7.3 +structlog==25.5.0 +svcs==25.1.0 +tableauserverclient==0.40 +tabulate==0.10.0 +tenacity==9.1.4 +teradatasql==20.0.0.56 +teradatasqlalchemy==20.0.0.9 +termcolor==3.3.0 +text-unidecode==1.3 +textual==6.2.1 +threadpoolctl==3.6.0 +thrift-sasl==0.4.3 +thrift==0.16.0 +tiktoken==0.12.0 +tinycss2==1.4.0 +tokenizers==0.22.2 +tomlkit==0.14.0 +tornado==6.5.5 +tqdm==4.67.3 +traitlets==5.14.3 +trino==0.337.0 +typer==0.24.1 +types-protobuf==7.34.1.20260408 +types-requests==2.33.0.20260408 +typing-inspect==0.9.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2026.1 +tzlocal==5.3.1 +uc-micro-py==2.0.0 +universal_pathlib==0.3.10 +uritemplate==4.2.0 +urllib3==2.6.3 +uuid6==2025.0.1 +uv==0.11.7 +uvicorn==0.44.0 +uvloop==0.22.1 +validators==0.35.0 +vertica-python==1.4.0 +vine==5.1.0 +virtualenv==21.2.4 +watchfiles==1.1.1 +watchtower==3.4.0 +wcwidth==0.6.0 +weaviate-client==4.16.2 +webencodings==0.5.1 +websocket-client==1.8.0 +websockets==16.0 +wheel==0.46.3 +wirerope==1.0.0 +wrapt==2.1.2 +xmlsec==1.3.17 +xmltodict==1.0.4 +yandex-query-client==0.1.4 +yandexcloud==0.386.0 +yarl==1.23.0 +ydb-dbapi==0.1.20 +ydb==3.28.0 +zeep==4.3.2 +zenpy==2.0.57 +zipp==3.23.1 +zope.event==6.1 +zope.interface==8.3 +zstandard==0.25.0 diff --git a/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch b/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch new file mode 100644 index 000000000..d62c0607a --- /dev/null +++ b/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch @@ -0,0 +1,71 @@ +From 271099b3cff14cb951523050e367b3c3222ead16 Mon Sep 17 00:00:00 2001 +From: Siegfried Weber +Date: Mon, 9 Mar 2026 10:11:28 +0100 +Subject: Allow overriding the logging configuration + +--- + .../src/airflow_shared/logging/structlog.py | 30 ++++++++++++++++++- + 1 file changed, 29 insertions(+), 1 deletion(-) + +diff --git a/shared/logging/src/airflow_shared/logging/structlog.py b/shared/logging/src/airflow_shared/logging/structlog.py +index d5b0b9a8bfe..649451a5502 100644 +--- a/shared/logging/src/airflow_shared/logging/structlog.py ++++ b/shared/logging/src/airflow_shared/logging/structlog.py +@@ -436,6 +436,18 @@ def structlog_processors( + return shared_processors, console, console + + ++def update_config( ++ config: dict, ++ updates: Mapping, ++): ++ for key, value in updates.items(): ++ if isinstance(value, Mapping): ++ config[key] = update_config(config.get(key, {}), value) ++ else: ++ config[key] = value ++ return config ++ ++ + def configure_logging( + *, + json_output: bool = False, +@@ -518,6 +530,19 @@ def configure_logging( + else: + PER_LOGGER_LEVELS[log] = loglevel + ++ # Use the log levels defined in stdlib_config for structlog ++ logger_configs = stdlib_config.get("loggers", {}) ++ logger_configs[""] = stdlib_config.get("root", {}) ++ for logger, logger_config in logger_configs.items(): ++ if "level" in logger_config: ++ loglevel = logger_config["level"] ++ if isinstance(loglevel, str): ++ try: ++ loglevel = NAME_TO_LEVEL[loglevel.lower()] ++ except KeyError: ++ raise ValueError(f"Invalid log level for logger {logger!r}: {loglevel!r}") from None ++ PER_LOGGER_LEVELS[logger] = loglevel ++ + shared_pre_chain, for_stdlib, for_structlog = structlog_processors( + json_output, + log_format=log_format, +@@ -564,7 +589,7 @@ def configure_logging( + + import logging.config + +- config = {**stdlib_config} ++ config = {} + config.setdefault("version", 1) + config.setdefault("disable_existing_loggers", False) + config["formatters"] = {**config.get("formatters", {})} +@@ -617,6 +642,9 @@ def configure_logging( + "propagate": True, + } + ++ # Merge stdlib_config into config and override existing values ++ update_config(config, stdlib_config) ++ + logging.config.dictConfig(config) + + if json_output: diff --git a/airflow/stackable/patches/3.2.1/patchable.toml b/airflow/stackable/patches/3.2.1/patchable.toml new file mode 100644 index 000000000..dddbef2b6 --- /dev/null +++ b/airflow/stackable/patches/3.2.1/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/airflow.git" +base = "e5968b88e45f2af9d1047a51188355af7cbe79f4" diff --git a/shell.nix b/shell.nix index ed1742207..a13fa9974 100644 --- a/shell.nix +++ b/shell.nix @@ -10,6 +10,8 @@ pkgs.mkShell { nodejs zizmor git-cliff + python312 + uv ]; buildInputs = with pkgs; [