diff --git a/CHANGELOG.md b/CHANGELOG.md index ba0aad35..b10c460f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +### Fixed + +- BREAKING: Fixed various issues in the CRD structure. `clusterConfig.credentialsSecret` is now mandatory ([#353]). + +[#353]: https://github.com/stackabletech/airflow-operator/pull/353 + ## [23.11.0] - 2023-11-24 ### Added diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 536e5a97..5d3eb668 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -6930,11 +6930,6 @@ spec: - roleGroups type: object clusterConfig: - default: - authentication: [] - credentialsSecret: '' - dagsGitSync: [] - listenerClass: cluster-internal description: Global cluster configuration that applies to all roles and role groups properties: authentication: @@ -7000,7 +6995,7 @@ spec: type: object type: array exposeConfig: - nullable: true + default: false type: boolean listenerClass: default: cluster-internal @@ -7020,7 +7015,7 @@ spec: - external-stable type: string loadExamples: - nullable: true + default: false type: boolean vectorAggregatorConfigMapName: description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. @@ -24940,6 +24935,7 @@ spec: - roleGroups type: object required: + - clusterConfig - image type: object status: diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 07ba44c2..4cdc32de 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -63,9 +63,8 @@ mod tests { spec: image: productVersion: 2.7.2 - loadExamples: true - exposeConfig: false - credentialsSecret: simple-airflow-credentials + clusterConfig: + credentialsSecret: airflow-credentials webservers: roleGroups: default: @@ -160,9 +159,8 @@ mod tests { spec: image: productVersion: 2.7.2 - loadExamples: true - exposeConfig: false - credentialsSecret: simple-airflow-credentials + clusterConfig: + credentialsSecret: airflow-credentials webservers: roleGroups: default: diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index 93d799c7..777c6758 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -39,7 +39,7 @@ pub struct AirflowAuthenticationConfigResolved { pub sync_roles_at: FlaskRolesSyncMoment, } -#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AirflowAuthentication { /// The Airflow authentication settings. diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 353872f4..b36cb70e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -151,32 +151,41 @@ impl FlaskAppConfigOptions for AirflowConfigOptions { pub struct AirflowClusterSpec { /// The Airflow image to use pub image: ProductImage, + /// Global cluster configuration that applies to all roles and role groups - #[serde(default)] pub cluster_config: AirflowClusterConfig, + /// Cluster operations like pause reconciliation or cluster stop. #[serde(default)] pub cluster_operation: ClusterOperation, + #[serde(default, skip_serializing_if = "Option::is_none")] pub webservers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub schedulers: Option>, + #[serde(flatten)] pub executor: AirflowExecutor, } -#[derive(Clone, Deserialize, Default, Debug, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AirflowClusterConfig { #[serde(flatten)] pub authentication: AirflowAuthentication, + pub credentials_secret: String, + #[serde(default)] pub dags_git_sync: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub expose_config: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub load_examples: Option, + + #[serde(default)] + pub expose_config: bool, + + #[serde(default)] + pub load_examples: bool, + /// This field controls which type of Service the Operator creates for this AirflowCluster: /// /// * cluster-internal: Use a ClusterIP service @@ -190,12 +199,15 @@ pub struct AirflowClusterConfig { /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. #[serde(default)] pub listener_class: CurrentlySupportedListenerClasses, + /// Name of the Vector aggregator discovery ConfigMap. /// It must contain the key `ADDRESS` with the address of the Vector aggregator. #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub volumes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub volume_mounts: Option>, } @@ -207,8 +219,10 @@ pub enum CurrentlySupportedListenerClasses { #[default] #[serde(rename = "cluster-internal")] ClusterInternal, + #[serde(rename = "external-unstable")] ExternalUnstable, + #[serde(rename = "external-stable")] ExternalStable, } @@ -263,6 +277,7 @@ pub struct Connections { pub enum AirflowRole { #[strum(serialize = "webserver")] Webserver, + #[strum(serialize = "scheduler")] Scheduler, #[strum(serialize = "worker")] @@ -342,6 +357,7 @@ pub enum AirflowExecutor { #[serde(flatten)] config: Role, }, + #[serde(rename = "kubernetesExecutors")] KubernetesExecutor { #[serde(flatten)] @@ -726,16 +742,6 @@ pub fn build_recommended_labels<'a, T>( } } -/// A reference to a [`AirflowCluster`] -#[derive(Clone, Default, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AirflowClusterRef { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub namespace: Option, -} - #[cfg(test)] mod tests { use crate::AirflowCluster; @@ -777,7 +783,7 @@ mod tests { assert_eq!("2.7.2", &resolved_airflow_image.product_version); assert_eq!("KubernetesExecutor", cluster.spec.executor.to_string()); - assert!(cluster.spec.cluster_config.load_examples.unwrap_or(false)); - assert!(cluster.spec.cluster_config.expose_config.unwrap_or(false)); + assert!(cluster.spec.cluster_config.load_examples); + assert!(cluster.spec.cluster_config.expose_config); } } diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 4fcfda61..d4803455 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1157,7 +1157,7 @@ fn build_mapped_envs( }) } } - if let Some(true) = airflow.spec.cluster_config.load_examples { + if airflow.spec.cluster_config.load_examples { env.push(EnvVar { name: "AIRFLOW__CORE__LOAD_EXAMPLES".into(), value: Some("True".into()), @@ -1171,7 +1171,7 @@ fn build_mapped_envs( }) } - if let Some(true) = airflow.spec.cluster_config.expose_config { + if airflow.spec.cluster_config.expose_config { env.push(EnvVar { name: "AIRFLOW__WEBSERVER__EXPOSE_CONFIG".into(), value: Some("True".into()),