diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b51aea..548e7b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ This means you need to replace your simple database connection string with a typed struct. This struct is consistent between different CRDs, so that you can easily copy/paste it between stacklets. Read on the [Airflow database documentation](https://docs.stackable.tech/home/nightly/airflow/usage-guide/database-connections) for details ([#754]). +- BREAKING: Renamed and moved the `celeryExecutor` broker and results backend to `clusterConfig` ([#786]). + The results backend `spec.celeryExecutors.resultBackend` is now `spec.clusterConfig.celeryResultsBackend`. + The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`. ### Fixed @@ -33,6 +36,7 @@ [#777]: https://github.com/stackabletech/airflow-operator/pull/777 [#754]: https://github.com/stackabletech/airflow-operator/pull/754 [#784]: https://github.com/stackabletech/airflow-operator/pull/784 +[#786]: https://github.com/stackabletech/airflow-operator/pull/786 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.nix b/Cargo.nix index 32f0dd35..5ecf9a54 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4868,7 +4868,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "k8s_version"; authors = [ @@ -9643,7 +9643,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_certs"; authors = [ @@ -9746,7 +9746,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_operator"; authors = [ @@ -9926,7 +9926,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9961,7 +9961,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_shared"; authors = [ @@ -10042,7 +10042,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_telemetry"; authors = [ @@ -10152,7 +10152,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_versioned"; authors = [ @@ -10202,7 +10202,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10270,7 +10270,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; + sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 4ab79fa6..71fbc1c3 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/docs/modules/airflow/examples/example-airflow-incluster.yaml b/docs/modules/airflow/examples/example-airflow-incluster.yaml index 2ae9327a..047fee95 100644 --- a/docs/modules/airflow/examples/example-airflow-incluster.yaml +++ b/docs/modules/airflow/examples/example-airflow-incluster.yaml @@ -15,6 +15,15 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials webservers: roleConfig: listenerClass: external-unstable @@ -29,15 +38,6 @@ spec: envOverrides: *envOverrides replicas: 1 celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials roleGroups: default: envOverrides: *envOverrides diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index 0c95fb3b..8697754e 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -16,22 +16,22 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials - webservers: - roleConfig: - listenerClass: external-unstable - roleGroups: - default: - replicas: 1 - celeryExecutors: - resultBackend: + celeryResultsBackend: postgresql: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials - broker: + celeryBroker: redis: host: airflow-redis-master credentialsSecretName: airflow-redis-credentials + webservers: + roleConfig: + listenerClass: external-unstable + roleGroups: + default: + replicas: 1 + celeryExecutors: roleGroups: default: replicas: 1 diff --git a/docs/modules/airflow/pages/usage-guide/database-connections.adoc b/docs/modules/airflow/pages/usage-guide/database-connections.adoc index 80d15cc8..12897a06 100644 --- a/docs/modules/airflow/pages/usage-guide/database-connections.adoc +++ b/docs/modules/airflow/pages/usage-guide/database-connections.adoc @@ -20,22 +20,26 @@ spec: <2> A reference to a Secret which must contain the two fields `username` and `password`. The queue/broker metadata and URL is only needed when running the celery executor. -The `resultBackend` definition uses the same structure as `metadataDatabase` shown above. -The `broker` definition requires Redis connection details. +The `celeryResultsBackend` definition uses the same structure as `metadataDatabase` shown above. +The `celeryBroker` definition requires Redis connection details. [source,yaml] ---- spec: - celeryExecutors: - resultBackend: + clusterConfig: + celeryResultsBackend: postgresql: # <1> host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials # <2> - broker: + celeryBroker: redis: # <3> host: airflow-redis-master credentialsSecretName: airflow-redis-credentials # <2> + celeryExecutors: + roleGroups: + default: + replicas: 1 ---- <1> A reference to one of the supported database backends (e.g. `postgresql`). <2> A reference to a secret which must contain the two fields `username` and `password`. @@ -57,10 +61,11 @@ spec: [source,yaml] ---- spec: - resultBackend: + clusterConfig: + celeryResultsBackend: generic: connectionUrlSecretName: postgresql-celery # <2> - broker: + celeryBroker: generic: connectionUrlSecretName: redis-celery # <3> ---- diff --git a/extra/crds.yaml b/extra/crds.yaml index f195a1bf..9ec7978a 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -39,66 +39,6 @@ spec: The celery executor. Deployed with an explicit number of replicas. properties: - broker: - description: Connection information for the celery broker queue. - oneOf: - - required: - - redis - - required: - - generic - properties: - generic: - description: |- - A generic Celery database connection for broker or result backend types not covered by a - dedicated variant. - - Use this when you need a Celery-compatible connection that does not have a first-class - connection type. The complete connection URL is read from a Secret, giving the user full - control over the connection string. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. - type: string - required: - - connectionUrlSecretName - type: object - redis: - description: |- - Connection settings for a [Redis](https://redis.io/) instance. - - Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). - properties: - credentialsSecretName: - description: |- - Name of a Secret containing the `username` and `password` keys used to authenticate - against the Redis server. - type: string - databaseId: - default: 0 - description: |- - Numeric index of the Redis logical database to use. Defaults to `0`. - - Redis supports multiple logical databases within a single instance, identified by an - integer index. Database `0` is the default. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - host: - description: Hostname or IP address of the Redis server. - type: string - port: - default: 6379 - description: Port the Redis server is listening on. Defaults to `6379`. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecretName - - host - type: object - type: object cliOverrides: additionalProperties: type: string @@ -575,65 +515,6 @@ spec: for more information. type: object x-kubernetes-preserve-unknown-fields: true - resultBackend: - description: Connection information for the celery backend database. - oneOf: - - required: - - postgresql - - required: - - generic - properties: - generic: - description: |- - A generic Celery database connection for broker or result backend types not covered by a - dedicated variant. - - Use this when you need a Celery-compatible connection that does not have a first-class - connection type. The complete connection URL is read from a Secret, giving the user full - control over the connection string. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. - type: string - required: - - connectionUrlSecretName - type: object - postgresql: - description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. - properties: - credentialsSecretName: - description: |- - Name of a Secret containing the `username` and `password` keys used to authenticate - against the PostgreSQL server. - type: string - database: - description: Name of the database (schema) to connect to. - type: string - host: - description: Hostname or IP address of the PostgreSQL server. - type: string - parameters: - additionalProperties: - type: string - default: {} - description: |- - Additional map of JDBC connection parameters to append to the connection URL. The given - `HashMap` will be converted to query parameters in the form of - `?param1=value1¶m2=value2`. - type: object - port: - default: 5432 - description: Port the PostgreSQL server is listening on. Defaults to `5432`. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecretName - - database - - host - type: object - type: object roleConfig: default: podDisruptionBudget: @@ -1161,8 +1042,6 @@ spec: type: object type: object required: - - broker - - resultBackend - roleGroups type: object clusterConfig: @@ -1276,6 +1155,127 @@ spec: - configMapName type: object type: object + celeryBroker: + description: Connection information for the celery broker queue. + nullable: true + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the Redis server. + type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + host: + description: Hostname or IP address of the Redis server. + type: string + port: + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - host + type: object + type: object + celeryResultsBackend: + description: Connection information for the celery backend database. + nullable: true + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object credentialsSecretName: description: |- The name of the Secret object containing the admin user credentials. Read the @@ -6229,66 +6229,6 @@ spec: The celery executor. Deployed with an explicit number of replicas. properties: - broker: - description: Connection information for the celery broker queue. - oneOf: - - required: - - redis - - required: - - generic - properties: - generic: - description: |- - A generic Celery database connection for broker or result backend types not covered by a - dedicated variant. - - Use this when you need a Celery-compatible connection that does not have a first-class - connection type. The complete connection URL is read from a Secret, giving the user full - control over the connection string. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. - type: string - required: - - connectionUrlSecretName - type: object - redis: - description: |- - Connection settings for a [Redis](https://redis.io/) instance. - - Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). - properties: - credentialsSecretName: - description: |- - Name of a Secret containing the `username` and `password` keys used to authenticate - against the Redis server. - type: string - databaseId: - default: 0 - description: |- - Numeric index of the Redis logical database to use. Defaults to `0`. - - Redis supports multiple logical databases within a single instance, identified by an - integer index. Database `0` is the default. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - host: - description: Hostname or IP address of the Redis server. - type: string - port: - default: 6379 - description: Port the Redis server is listening on. Defaults to `6379`. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecretName - - host - type: object - type: object cliOverrides: additionalProperties: type: string @@ -6765,65 +6705,6 @@ spec: for more information. type: object x-kubernetes-preserve-unknown-fields: true - resultBackend: - description: Connection information for the celery backend database. - oneOf: - - required: - - postgresql - - required: - - generic - properties: - generic: - description: |- - A generic Celery database connection for broker or result backend types not covered by a - dedicated variant. - - Use this when you need a Celery-compatible connection that does not have a first-class - connection type. The complete connection URL is read from a Secret, giving the user full - control over the connection string. - properties: - connectionUrlSecretName: - description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. - type: string - required: - - connectionUrlSecretName - type: object - postgresql: - description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. - properties: - credentialsSecretName: - description: |- - Name of a Secret containing the `username` and `password` keys used to authenticate - against the PostgreSQL server. - type: string - database: - description: Name of the database (schema) to connect to. - type: string - host: - description: Hostname or IP address of the PostgreSQL server. - type: string - parameters: - additionalProperties: - type: string - default: {} - description: |- - Additional map of JDBC connection parameters to append to the connection URL. The given - `HashMap` will be converted to query parameters in the form of - `?param1=value1¶m2=value2`. - type: object - port: - default: 5432 - description: Port the PostgreSQL server is listening on. Defaults to `5432`. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecretName - - database - - host - type: object - type: object roleConfig: default: podDisruptionBudget: @@ -7351,8 +7232,6 @@ spec: type: object type: object required: - - broker - - resultBackend - roleGroups type: object clusterConfig: @@ -7466,6 +7345,127 @@ spec: - configMapName type: object type: object + celeryBroker: + description: Connection information for the celery broker queue. + nullable: true + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + redis: + description: |- + Connection settings for a [Redis](https://redis.io/) instance. + + Redis is commonly used as a Celery message broker or result backend (e.g. for Apache Airflow). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the Redis server. + type: string + databaseId: + default: 0 + description: |- + Numeric index of the Redis logical database to use. Defaults to `0`. + + Redis supports multiple logical databases within a single instance, identified by an + integer index. Database `0` is the default. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + host: + description: Hostname or IP address of the Redis server. + type: string + port: + default: 6379 + description: Port the Redis server is listening on. Defaults to `6379`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - host + type: object + type: object + celeryResultsBackend: + description: Connection information for the celery backend database. + nullable: true + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + description: |- + A generic Celery database connection for broker or result backend types not covered by a + dedicated variant. + + Use this when you need a Celery-compatible connection that does not have a first-class + connection type. The complete connection URL is read from a Secret, giving the user full + control over the connection string. + properties: + connectionUrlSecretName: + description: The name of the Secret that contains an `connectionUrl` key with the complete Celery URL. + type: string + required: + - connectionUrlSecretName + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object credentialsSecretName: description: |- The name of the Secret object containing the admin user credentials. Read the diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 25a5bc71..54671bb3 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -36,10 +36,7 @@ use stackable_operator::{ random_secret_creation, rbac::build_rbac_resources, }, - crd::{ - authentication::{core as auth_core, ldap}, - git_sync, listener, - }, + crd::{authentication::ldap, git_sync, listener}, database_connections::{ TemplatingMechanism, drivers::{ @@ -136,11 +133,6 @@ pub enum Error { #[snafu(display("object defines no airflow config role"))] NoAirflowRole, - #[snafu(display("failed to apply global Service"))] - ApplyRoleService { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to apply Service for {rolegroup}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, @@ -189,21 +181,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to retrieve AuthenticationClass {authentication_class}"))] - AuthenticationClassRetrieval { - source: stackable_operator::cluster_resources::Error, - authentication_class: ObjectRef, - }, - - #[snafu(display( - "Airflow doesn't support the AuthenticationClass provider - {authentication_class_provider} from AuthenticationClass {authentication_class}" - ))] - AuthenticationClassProviderNotSupported { - authentication_class_provider: String, - authentication_class: ObjectRef, - }, - #[snafu(display("failed to build config file for {rolegroup}"))] BuildRoleGroupConfigFile { source: FlaskAppConfigWriterError, @@ -291,11 +268,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display( - "failed to build volume or volume mount spec for the LDAP backend TLS config" - ))] - VolumeAndMounts { source: ldap::v1alpha1::Error }, - #[snafu(display("failed to construct config"))] ConstructConfig { source: config::Error }, @@ -421,22 +393,35 @@ pub async fn reconcile_airflow( .cluster_config .metadata_database .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); - let celery_database_connection_details = match &airflow.spec.executor { - AirflowExecutor::CeleryExecutors { - result_backend: celery_result_backend, - broker: celery_broker, - .. - } => { - let celery_result_backend = celery_result_backend - .celery_connection_details_with_templating( - "CELERY_RESULT_BACKEND", - &templating_mechanism, - ); - let celery_broker = celery_broker - .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); - Some((celery_result_backend, celery_broker)) + + let celery_database_connection_details = if let ( + Some(celery_results_backend), + Some(celery_broker), + ) = ( + &airflow.spec.cluster_config.celery_results_backend, + &airflow.spec.cluster_config.celery_broker, + ) { + // The celery results backend and celery broker only work with configured celeryExecutors. + // Emit a warning if celery executors were not configured properly. + if !matches!( + &airflow.spec.executor, + AirflowExecutor::CeleryExecutors { .. } + ) { + tracing::warn!( + "No `spec.celeryExecutors` configured, but `spec.clusterConfig.celeryResultsBackend` and `spec.clusterConfig.celeryBroker` are provided. This only works in combination with a celery executor!" + ) } - _ => None, + + let celery_results_backend = celery_results_backend + .celery_connection_details_with_templating( + "CELERY_RESULT_BACKEND", + &templating_mechanism, + ); + let celery_broker = celery_broker + .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); + Some((celery_results_backend, celery_broker)) + } else { + None }; let mut roles = HashMap::new(); diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 6ad8c1eb..d9331e76 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -77,20 +77,20 @@ mod tests { host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials - webservers: - roleGroups: - default: - replicas: 1 - celeryExecutors: - resultBackend: + celeryResultsBackend: postgresql: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials - broker: + celeryBroker: redis: host: airflow-redis-master credentialsSecretName: airflow-redis-credentials + webservers: + roleGroups: + default: + replicas: 1 + celeryExecutors: roleGroups: default: replicas: 2 diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f0fc0f27..3e264113 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -309,6 +309,14 @@ pub mod versioned { #[serde(default)] pub database_initialization: DatabaseInitializationConfig, + #[serde(skip_serializing_if = "Option::is_none")] + /// Connection information for the celery backend database. + pub celery_results_backend: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + /// Connection information for the celery broker queue. + pub celery_broker: Option, + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) @@ -857,12 +865,6 @@ pub enum AirflowExecutor { CeleryExecutors { #[serde(flatten)] config: Box, - - /// Connection information for the celery backend database. - result_backend: CeleryResultBackendConnection, - - /// Connection information for the celery broker queue. - broker: CeleryBrokerConnection, }, /// With the Kubernetes executor, executor Pods are created on demand. @@ -1243,6 +1245,15 @@ mod tests { host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials authentication: - authenticationClass: my-ldap userRegistrationRole: Admin @@ -1298,15 +1309,6 @@ mod tests { FILE_HEADER: | COMMON_HEADER_VAR = "group-value" celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: true diff --git a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 index 20d3fef4..0eaaafb3 100644 --- a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 @@ -55,6 +55,15 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials webservers: roleConfig: listenerClass: external-unstable @@ -65,15 +74,6 @@ spec: default: replicas: 1 celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 index f52378dc..e689d94b 100644 --- a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 @@ -48,6 +48,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} webservers: roleConfig: listenerClass: test-external-stable-$NAMESPACE @@ -63,15 +74,6 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 index d818b52d..cb45682a 100644 --- a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 @@ -71,6 +71,17 @@ commands: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} authentication: - authenticationClass: {% if test_scenario['values']['ldap-authentication'] == 'no-tls' -%} no-tls-$NAMESPACE @@ -92,15 +103,6 @@ commands: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index 9fddf14b..d93a44da 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -96,6 +96,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} dagsGitSync: - repo: https://github.com/stackabletech/example-dags gitFolder: dags @@ -162,15 +173,6 @@ spec: configMap: airflow-log-config {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: resources: cpu: diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index 83d54a78..67533dbd 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -23,9 +23,9 @@ def check_sent_events(): }, ) - assert ( - response.status_code == 200 - ), "Cannot access the API of the vector aggregator." + assert response.status_code == 200, ( + "Cannot access the API of the vector aggregator." + ) result = response.json() @@ -35,13 +35,13 @@ def check_sent_events(): componentId = transform["componentId"] if componentId == "filteredInvalidEvents": - assert ( - sentEvents is None or sentEvents["sentEventsTotal"] == 0 - ), "Invalid log events were sent." + assert sentEvents is None or sentEvents["sentEventsTotal"] == 0, ( + "Invalid log events were sent." + ) else: - assert ( - sentEvents is not None and sentEvents["sentEventsTotal"] > 0 - ), f'No events were sent in "{componentId}".' + assert sentEvents is not None and sentEvents["sentEventsTotal"] > 0, ( + f'No events were sent in "{componentId}".' + ) if __name__ == "__main__": diff --git a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 index b4fb90d3..71fba79d 100644 --- a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 @@ -93,6 +93,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} volumes: - name: test-cm-dag configMap: @@ -114,15 +125,6 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 3ea3d9a3..380b61f9 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -92,6 +92,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} dagsGitSync: {% if test_scenario['values']['access'] == 'ssh' %} - repo: ssh://git@github.com/stackable-airflow/dags.git @@ -132,15 +143,6 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 index 20d3fef4..0eaaafb3 100644 --- a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 @@ -55,6 +55,15 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials webservers: roleConfig: listenerClass: external-unstable @@ -65,15 +74,6 @@ spec: default: replicas: 1 celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 index aca2981a..9907a482 100644 --- a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 @@ -61,6 +61,15 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials webservers: roleConfig: listenerClass: external-unstable @@ -75,15 +84,6 @@ spec: COMMON_VAR: group-value # overrides role value GROUP_VAR: group-value # only defined here at group level celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials envOverrides: COMMON_VAR: role-value # overridden by role group below ROLE_VAR: role-value # only defined here at role level diff --git a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 index c61ddbe5..04f0dbc2 100644 --- a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 @@ -55,6 +55,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} webservers: roleConfig: listenerClass: external-unstable @@ -68,15 +79,6 @@ spec: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_conn {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 index bfc1fde8..0bdcdc99 100644 --- a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 @@ -55,6 +55,15 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials webservers: roleConfig: listenerClass: external-unstable @@ -65,15 +74,6 @@ spec: default: replicas: 1 celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index 2b4852f7..dcc6e492 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -58,6 +58,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} webservers: roleConfig: listenerClass: external-unstable @@ -80,15 +91,6 @@ spec: COMMON_HEADER_VAR = "group-value" {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 index e82d79bc..8eff34ad 100644 --- a/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 @@ -115,6 +115,17 @@ spec: host: airflow-postgresql database: airflow credentialsSecretName: airflow-postgresql-credentials +{% if test_scenario['values']['executor'] == 'celery' %} + celeryResultsBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + celeryBroker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials +{% endif %} volumes: - name: triggerer-dag configMap: @@ -133,15 +144,6 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: - resultBackend: - postgresql: - host: airflow-postgresql - database: airflow - credentialsSecretName: airflow-postgresql-credentials - broker: - redis: - host: airflow-redis-master - credentialsSecretName: airflow-redis-credentials roleGroups: default: envOverrides: *envOverrides