diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 33da1d48eb972..a04bd12d5e22d 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -17,20 +17,20 @@ under the License. */}} --- -{{- $nodeSelector := or .Values.workers.kubernetes.nodeSelector .Values.workers.nodeSelector .Values.nodeSelector }} -{{- $affinity := or .Values.workers.kubernetes.affinity .Values.workers.affinity .Values.affinity }} -{{- $tolerations := or .Values.workers.kubernetes.tolerations .Values.workers.tolerations .Values.tolerations }} -{{- $topologySpreadConstraints := or .Values.workers.kubernetes.topologySpreadConstraints .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} -{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} -{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosSidecar .Values.workers.kerberosSidecar .Values) }} -{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kubernetes.kerberosSidecar.containerLifecycleHooks .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosInitContainer .Values.workers.kerberosInitContainer .Values) }} -{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.kubernetes.kerberosInitContainer.containerLifecycleHooks .Values.workers.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} -{{- $containerLifecycleHooks := or .Values.workers.kubernetes.containerLifecycleHooks .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (or .Values.workers.kubernetes.safeToEvict (and (not (has .Values.workers.kubernetes.safeToEvict (list true false))) .Values.workers.safeToEvict) | toString) }} -{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict (.Values.workers.kubernetes.podAnnotations | default .Values.workers.podAnnotations) }} -{{- $schedulerName := or .Values.workers.kubernetes.schedulerName .Values.workers.schedulerName .Values.schedulerName }} +{{- $nodeSelector := or .Values.workers.kubernetes.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.workers.kubernetes.affinity .Values.affinity }} +{{- $tolerations := or .Values.workers.kubernetes.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.workers.kubernetes.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers.kubernetes .Values) }} +{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosSidecar .Values) }} +{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kubernetes.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosInitContainer .Values) }} +{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.kubernetes.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers.kubernetes .Values) }} +{{- $containerLifecycleHooks := or .Values.workers.kubernetes.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (.Values.workers.kubernetes.safeToEvict | toString) }} +{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.kubernetes.podAnnotations }} +{{- $schedulerName := or .Values.workers.kubernetes.schedulerName .Values.schedulerName }} apiVersion: v1 kind: Pod metadata: @@ -39,12 +39,12 @@ metadata: tier: airflow component: worker release: {{ .Release.Name }} - {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} - {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.kubernetes.labels }} + {{- mustMerge .Values.workers.kubernetes.labels .Values.labels | toYaml | nindent 4 }} {{- end }} annotations: {{- tpl (toYaml $podAnnotations) . | nindent 4 }} - {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + {{- if .Values.workers.kubernetes.kerberosInitContainer.enabled }} checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }} {{- end }} spec: @@ -52,10 +52,10 @@ spec: {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }} {{- end }} - {{- if or .Values.workers.kubernetes.extraInitContainers .Values.workers.extraInitContainers }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraInitContainers | default .Values.workers.extraInitContainers)) . | nindent 4 }} + {{- if .Values.workers.kubernetes.extraInitContainers }} + {{- tpl (toYaml .Values.workers.kubernetes.extraInitContainers) . | nindent 4 }} {{- end }} - {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + {{- if .Values.workers.kubernetes.kerberosInitContainer.enabled }} - name: kerberos-init image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -64,7 +64,7 @@ spec: lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosInitContainer) . | nindent 8 }} {{- end }} args: ["kerberos", "-o"] - resources: {{- toYaml (.Values.workers.kubernetes.kerberosInitContainer.resources | default .Values.workers.kerberosInitContainer.resources) | nindent 8 }} + resources: {{- toYaml .Values.workers.kubernetes.kerberosInitContainer.resources | nindent 8 }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} @@ -86,8 +86,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- if .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.kubernetes.extraVolumeMounts) . | nindent 8 }} {{- end }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} {{- include "airflow_api_server_config_mount" . | nindent 8 }} @@ -106,7 +106,7 @@ spec: env: - name: AIRFLOW__CORE__EXECUTOR value: {{ .Values.executor | quote }} - {{- if or .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} + {{- if or .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kubernetes.kerberosInitContainer.enabled }} - name: KRB5_CONFIG value: {{ .Values.kerberos.configPath | quote }} - name: KRB5CCNAME @@ -114,7 +114,7 @@ spec: {{- end }} {{- include "standard_airflow_environment" . | indent 6}} {{- include "custom_airflow_environment" . | indent 6 }} - {{- include "container_extra_envs" (list . (.Values.workers.kubernetes.env | default .Values.workers.env)) | indent 6 }} + {{- include "container_extra_envs" (list . .Values.workers.kubernetes.env) | indent 6 }} image: {{ template "pod_template_image" . }} imagePullPolicy: {{ .Values.images.pod_template.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 8 }} @@ -122,10 +122,10 @@ spec: lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 8 }} {{- end }} name: base - {{- if or .Values.workers.kubernetes.command .Values.workers.command }} - command: {{ tpl (toYaml (.Values.workers.kubernetes.command | default .Values.workers.command)) . | nindent 8 }} + {{- if .Values.workers.kubernetes.command }} + command: {{ tpl (toYaml .Values.workers.kubernetes.command) . | nindent 8 }} {{- end }} - resources: {{- toYaml (.Values.workers.kubernetes.resources | default .Values.workers.resources) | nindent 8 }} + resources: {{- toYaml .Values.workers.kubernetes.resources | nindent 8 }} volumeMounts: - mountPath: {{ template "airflow_logs" . }} name: logs @@ -139,8 +139,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- if .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.kubernetes.extraVolumeMounts) . | nindent 8 }} {{- end }} {{- if .Values.kerberos.enabled }} - name: kerberos-keytab @@ -155,7 +155,7 @@ spec: mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} readOnly: true {{- end }} - {{- if or .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled }} + {{- if .Values.workers.kubernetes.kerberosSidecar.enabled }} - name: worker-kerberos image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -164,7 +164,7 @@ spec: lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 8 }} {{- end }} args: ["kerberos"] - resources: {{- toYaml (.Values.workers.kubernetes.kerberosSidecar.resources | default .Values.workers.kerberosSidecar.resources) | nindent 8 }} + resources: {{- toYaml .Values.workers.kubernetes.kerberosSidecar.resources | nindent 8 }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} @@ -186,8 +186,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} + {{- if .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.kubernetes.extraVolumeMounts) . | nindent 8 }} {{- end }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} {{- include "airflow_api_server_config_mount" . | nindent 8 }} @@ -201,18 +201,18 @@ spec: {{- include "custom_airflow_environment" . | indent 6 }} {{- include "standard_airflow_environment" . | indent 6 }} {{- end }} - {{- if or .Values.workers.kubernetes.extraContainers .Values.workers.extraContainers }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraContainers | default .Values.workers.extraContainers)) . | nindent 4 }} + {{- if .Values.workers.kubernetes.extraContainers }} + {{- tpl (toYaml .Values.workers.kubernetes.extraContainers) . | nindent 4 }} {{- end }} - {{- if or .Values.workers.kubernetes.priorityClassName .Values.workers.priorityClassName }} - priorityClassName: {{ .Values.workers.kubernetes.priorityClassName | default .Values.workers.priorityClassName }} + {{- if .Values.workers.kubernetes.priorityClassName }} + priorityClassName: {{ .Values.workers.kubernetes.priorityClassName }} {{- end }} - {{- if or .Values.workers.kubernetes.runtimeClassName .Values.workers.runtimeClassName }} - runtimeClassName: {{ .Values.workers.kubernetes.runtimeClassName | default .Values.workers.runtimeClassName }} + {{- if .Values.workers.kubernetes.runtimeClassName }} + runtimeClassName: {{ .Values.workers.kubernetes.runtimeClassName }} {{- end }} imagePullSecrets: {{- include "image_pull_secrets" . | nindent 4 }} - {{- if or .Values.workers.kubernetes.hostAliases .Values.workers.hostAliases }} - hostAliases: {{- toYaml (.Values.workers.kubernetes.hostAliases | default .Values.workers.hostAliases) | nindent 4 }} + {{- if .Values.workers.kubernetes.hostAliases }} + hostAliases: {{- toYaml .Values.workers.kubernetes.hostAliases | nindent 4 }} {{- end }} restartPolicy: Never securityContext: {{ $securityContext | nindent 4 }} @@ -221,14 +221,10 @@ spec: {{- if $schedulerName }} schedulerName: {{ $schedulerName }} {{- end }} - terminationGracePeriodSeconds: {{ .Values.workers.kubernetes.terminationGracePeriodSeconds | default .Values.workers.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.workers.kubernetes.terminationGracePeriodSeconds }} tolerations: {{- toYaml $tolerations | nindent 4 }} topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 4 }} - {{- if .Values.workers.kubernetes.serviceAccount.create }} serviceAccountName: {{ include "worker.kubernetes.serviceAccountName" . }} - {{- else }} - serviceAccountName: {{ include "worker.serviceAccountName" . }} - {{- end }} volumes: {{- if .Values.dags.persistence.enabled }} - name: dags @@ -252,7 +248,7 @@ spec: - configMap: name: {{ include "airflow_config" . }} name: config - {{- if and (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) (or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled .Values.workers.kubernetes.kerberosSidecar.enabled .Values.workers.kerberosSidecar.enabled) }} + {{- if and (or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName) (or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kubernetes.kerberosSidecar.enabled) }} - name: api-server-config configMap: name: {{ template "airflow_api_server_config_configmap_name" . }} @@ -267,6 +263,6 @@ spec: - name: kerberos-ccache emptyDir: {} {{- end }} - {{- if or .Values.workers.kubernetes.extraVolumes .Values.workers.extraVolumes }} - {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumes | default .Values.workers.extraVolumes)) . | nindent 2 }} + {{- if .Values.workers.kubernetes.extraVolumes }} + {{- tpl (toYaml .Values.workers.kubernetes.extraVolumes) . | nindent 2 }} {{- end }} diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 2ba8c2401e66d..94b6c89f4a500 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -218,718 +218,6 @@ DEPRECATION WARNING: {{- end }} -{{- if ne (int .Values.workers.replicas) 1 }} - - DEPRECATION WARNING: - `workers.replicas` has been renamed to `workers.celery.replicas`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.revisionHistoryLimit) }} - - DEPRECATION WARNING: - `workers.revisionHistoryLimit` has been renamed to `workers.celery.revisionHistoryLimit`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.command) }} - - DEPRECATION WARNING: - `workers.command` has been renamed to `workers.celery.command`/`workers.kubernetes.command`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (.Values.workers.args | toJson) (list "bash" "-c" "exec \\\nairflow celery worker\n{{- if and .Values.workers.queue (ne .Values.workers.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.queue }}\n{{- end }}" | toJson) }} - - DEPRECATION WARNING: - `workers.args` has been renamed to `workers.celery.args`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.livenessProbe.enabled }} - - DEPRECATION WARNING: - `workers.livenessProbe.enabled` has been renamed to `workers.celery.livenessProbe.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.livenessProbe.initialDelaySeconds) 10 }} - - DEPRECATION WARNING: - `workers.livenessProbe.initialDelaySeconds` has been renamed to `workers.celery.livenessProbe.initialDelaySeconds`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.livenessProbe.timeoutSeconds) 20 }} - - DEPRECATION WARNING: - `workers.livenessProbe.timeoutSeconds` has been renamed to `workers.celery.livenessProbe.timeoutSeconds`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.livenessProbe.failureThreshold) 5 }} - - DEPRECATION WARNING: - `workers.livenessProbe.failureThreshold` has been renamed to `workers.celery.livenessProbe.failureThreshold`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.livenessProbe.periodSeconds) 60 }} - - DEPRECATION WARNING: - `workers.livenessProbe.periodSeconds` has been renamed to `workers.celery.livenessProbe.periodSeconds`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.livenessProbe.command) }} - - DEPRECATION WARNING: - `workers.livenessProbe.command` has been renamed to `workers.celery.livenessProbe.command`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.updateStrategy) }} - - DEPRECATION WARNING: - `workers.updateStrategy` has been renamed to `workers.celery.updateStrategy`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (toJson .Values.workers.strategy | quote) (toJson "{\"rollingUpdate\":{\"maxSurge\":\"100%\",\"maxUnavailable\":\"50%\"}}") }} - - DEPRECATION WARNING: - `workers.strategy` has been renamed to `workers.celery.strategy`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.podManagementPolicy) }} - - DEPRECATION WARNING: - `workers.podManagementPolicy` has been renamed to `workers.celery.podManagementPolicy`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.securityContext) }} - - DEPRECATION WARNING: - `workers.securityContext` has been renamed to `workers.celery.securityContexts`/`workers.kubernetes.securityContexts`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.securityContexts.pod) }} - - DEPRECATION WARNING: - `workers.securityContexts.pod` has been renamed to `workers.celery.securityContexts.pod`/`workers.kubernetes.securityContexts.pod`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.securityContexts.container) }} - - DEPRECATION WARNING: - `workers.securityContexts.container` has been renamed to `workers.celery.securityContexts.container`/`workers.kubernetes.securityContexts.container`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.containerLifecycleHooks) }} - - DEPRECATION WARNING: - `workers.containerLifecycleHooks` has been renamed to `workers.celery.containerLifecycleHooks`/`workers.kubernetes.containerLifecycleHooks`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.podDisruptionBudget.enabled }} - - DEPRECATION WARNING: - `workers.podDisruptionBudget.enabled` has been renamed to `workers.celery.podDisruptionBudget.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.podDisruptionBudget.config.maxUnavailable) 1 }} - - DEPRECATION WARNING: - `workers.podDisruptionBudget.config.maxUnavailable` has been renamed to `workers.celery.podDisruptionBudget.config.maxUnavailable`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if hasKey .Values.workers.podDisruptionBudget.config "minAvailable" }} - - DEPRECATION WARNING: - `workers.podDisruptionBudget.config.minAvailable` has been renamed to `workers.celery.podDisruptionBudget.config.minAvailable`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.serviceAccount.automountServiceAccountToken }} - - DEPRECATION WARNING: - `workers.serviceAccount.automountServiceAccountToken` has been renamed to `workers.celery.serviceAccount.automountServiceAccountToken`/`workers.kubernetes.serviceAccount.automountServiceAccountToken`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.serviceAccount.create }} - - DEPRECATION WARNING: - `workers.serviceAccount.create` has been renamed to `workers.celery.serviceAccount.create`/`workers.kubernetes.serviceAccount.create`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.serviceAccount.name) }} - - DEPRECATION WARNING: - `workers.serviceAccount.name` has been renamed to `workers.celery.serviceAccount.name`/`workers.kubernetes.serviceAccount.name`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.serviceAccount.annotations) }} - - DEPRECATION WARNING: - `workers.serviceAccount.annotations` has been renamed to `workers.celery.serviceAccount.annotations`/`workers.kubernetes.serviceAccount.annotations`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.keda.enabled }} - - DEPRECATION WARNING: - `workers.keda.enabled` has been renamed to `workers.celery.keda.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.keda.namespaceLabels) }} - - DEPRECATION WARNING: - `workers.keda.namespaceLabels` has been renamed to `workers.celery.keda.namespaceLabels`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.keda.pollingInterval) 5 }} - - DEPRECATION WARNING: - `workers.keda.pollingInterval` has been renamed to `workers.celery.keda.pollingInterval`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.keda.cooldownPeriod) 30 }} - - DEPRECATION WARNING: - `workers.keda.cooldownPeriod` has been renamed to `workers.celery.keda.cooldownPeriod`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.keda.minReplicaCount) 0 }} - - DEPRECATION WARNING: - `workers.keda.minReplicaCount` has been renamed to `workers.celery.keda.minReplicaCount`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.keda.maxReplicaCount) 10 }} - - DEPRECATION WARNING: - `workers.keda.maxReplicaCount` has been renamed to `workers.celery.keda.maxReplicaCount`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.keda.advanced) }} - - DEPRECATION WARNING: - `workers.keda.advanced` has been renamed to `workers.celery.keda.advanced`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (quote .Values.workers.keda.query) (quote "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') AND queue IN ( {{- range $i, $q := splitList \",\" .Values.workers.queue -}} {{- if $i }},{{ end }}'{{ $q | trim }}' {{- end -}} ) {{- if contains \"KubernetesExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'KubernetesExecutor' {{- else if contains \"airflow.providers.edge3.executors.EdgeExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'EdgeExecutor' {{- end }}") }} - - DEPRECATION WARNING: - `workers.keda.query` has been renamed to `workers.celery.keda.query`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.keda.usePgbouncer }} - - DEPRECATION WARNING: - `workers.keda.usePgbouncer` has been renamed to `workers.celery.keda.usePgbouncer`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.hpa.enabled }} - - DEPRECATION WARNING: - `workers.hpa.enabled` has been renamed to `workers.celery.hpa.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.hpa.minReplicaCount) 0 }} - - DEPRECATION WARNING: - `workers.hpa.minReplicaCount` has been renamed to `workers.celery.hpa.minReplicaCount`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.hpa.maxReplicaCount) 5 }} - - DEPRECATION WARNING: - `workers.hpa.maxReplicaCount` has been renamed to `workers.celery.hpa.maxReplicaCount`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (toJson .Values.workers.hpa.metrics | quote) (toJson "[{\"resource\":{\"name\":\"cpu\",\"target\":{\"averageUtilization\":80,\"type\":\"Utilization\"}},\"type\":\"Resource\"}]") }} - - DEPRECATION WARNING: - `workers.hpa.metrics` has been renamed to `workers.celery.hpa.metrics`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.hpa.behavior) }} - - DEPRECATION WARNING: - `workers.hpa.behavior` has been renamed to `workers.celery.hpa.behavior`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.persistence.enabled }} - - DEPRECATION WARNING: - `workers.persistence.enabled` has been renamed to `workers.celery.persistence.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) }} - - DEPRECATION WARNING: - `workers.persistence.persistentVolumeClaimRetentionPolicy` has been renamed to `workers.celery.persistence.persistentVolumeClaimRetentionPolicy`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne .Values.workers.persistence.size "100Gi" }} - - DEPRECATION WARNING: - `workers.persistence.size` has been renamed to `workers.celery.persistence.size`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.persistence.storageClassName) }} - - DEPRECATION WARNING: - `workers.persistence.storageClassName` has been renamed to `workers.celery.persistence.storageClassName`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.persistence.fixPermissions }} - - DEPRECATION WARNING: - `workers.persistence.fixPermissions` has been renamed to `workers.celery.persistence.fixPermissions`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.persistence.annotations) }} - - DEPRECATION WARNING: - `workers.persistence.annotations` has been renamed to `workers.celery.persistence.annotations`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.persistence.securityContexts.container }} - - DEPRECATION WARNING: - `workers.persistence.securityContexts` has been renamed to `workers.celery.persistence.securityContexts`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosSidecar.enabled }} - - DEPRECATION WARNING: - `workers.kerberosSidecar.enabled` has been renamed to `workers.celery.kerberosSidecar.enabled`/`workers.kubernetes.kerberosSidecar.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosSidecar.resources }} - - DEPRECATION WARNING: - `workers.kerberosSidecar.resources` has been renamed to `workers.celery.kerberosSidecar.resources`/`workers.kubernetes.kerberosSidecar.resources`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosSidecar.securityContexts.container }} - - DEPRECATION WARNING: - `workers.kerberosSidecar.securityContexts.container` has been renamed to `workers.celery.kerberosSidecar.securityContexts.container`/`workers.kubernetes.kerberosSidecar.securityContexts.container`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosSidecar.containerLifecycleHooks }} - - DEPRECATION WARNING: - `workers.kerberosSidecar.containerLifecycleHooks` has been renamed to `workers.celery.kerberosSidecar.containerLifecycleHooks`/`workers.kubernetes.kerberosSidecar.containerLifecycleHooks`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosInitContainer.enabled }} - - DEPRECATION WARNING: - `workers.kerberosInitContainer.enabled` has been renamed to `workers.celery.kerberosInitContainer.enabled`/`workers.kubernetes.kerberosInitContainer.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosInitContainer.resources }} - - DEPRECATION WARNING: - `workers.kerberosInitContainer.resources` has been renamed to `workers.celery.kerberosInitContainer.resources`/`workers.kubernetes.kerberosInitContainer.resources`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosInitContainer.securityContexts.container }} - - DEPRECATION WARNING: - `workers.kerberosInitContainer.securityContexts.container` has been renamed to `workers.celery.kerberosInitContainer.securityContexts.container`/`workers.kubernetes.kerberosInitContainer.securityContexts.container`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.kerberosInitContainer.containerLifecycleHooks }} - - DEPRECATION WARNING: - `workers.kerberosInitContainer.containerLifecycleHooks` has been renamed to `workers.celery.kerberosInitContainer.containerLifecycleHooks`/`workers.kubernetes.kerberosInitContainer.containerLifecycleHooks`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.resources) }} - - DEPRECATION WARNING: - `workers.resources` has been renamed to `workers.celery.resources`/`workers.kubernetes.resources`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.terminationGracePeriodSeconds) 600 }} - - DEPRECATION WARNING: - `workers.terminationGracePeriodSeconds` has been renamed to `workers.celery.terminationGracePeriodSeconds`/`workers.kubernetes.terminationGracePeriodSeconds`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if .Values.workers.safeToEvict }} - - DEPRECATION WARNING: - `workers.safeToEvict` has been renamed to `workers.celery.safeToEvict`/`workers.kubernetes.safeToEvict`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.extraContainers) }} - - DEPRECATION WARNING: - `workers.extraContainers` has been renamed to `workers.celery.extraContainers`/`workers.kubernetes.extraContainers`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.extraInitContainers) }} - - DEPRECATION WARNING: - `workers.extraInitContainers` has been renamed to `workers.celery.extraInitContainers`/`workers.kubernetes.extraInitContainers`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.extraVolumes) }} - - DEPRECATION WARNING: - `workers.extraVolumes` has been renamed to `workers.celery.extraVolumes`/`workers.kubernetes.extraVolumes`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.extraVolumeMounts) }} - - DEPRECATION WARNING: - `workers.extraVolumeMounts` has been renamed to `workers.celery.extraVolumeMounts`/`workers.kubernetes.extraVolumeMounts`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.runtimeClassName) }} - - DEPRECATION WARNING: - `workers.runtimeClassName` has been renamed to `workers.celery.runtimeClassName`/`workers.kubernetes.runtimeClassName`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.priorityClassName) }} - - DEPRECATION WARNING: - `workers.priorityClassName` has been renamed to `workers.celery.priorityClassName`/`workers.kubernetes.priorityClassName`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.affinity) }} - - DEPRECATION WARNING: - `workers.affinity` has been renamed to `workers.celery.affinity`/`workers.kubernetes.affinity`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.tolerations) }} - - DEPRECATION WARNING: - `workers.tolerations` has been renamed to `workers.celery.tolerations`/`workers.kubernetes.tolerations`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.topologySpreadConstraints) }} - - DEPRECATION WARNING: - `workers.topologySpreadConstraints` has been renamed to `workers.celery.topologySpreadConstraints`/`workers.kubernetes.topologySpreadConstraints`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.nodeSelector) }} - - DEPRECATION WARNING: - `workers.nodeSelector` has been renamed to `workers.celery.nodeSelector`/`workers.kubernetes.nodeSelector`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.extraPorts) }} - - DEPRECATION WARNING: - `workers.extraPorts` has been renamed to `workers.celery.extraPorts`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.hostAliases) }} - - DEPRECATION WARNING: - `workers.hostAliases` has been renamed to `workers.celery.hostAliases`/`workers.kubernetes.hostAliases`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.annotations) }} - - DEPRECATION WARNING: - `workers.annotations` has been renamed to `workers.celery.annotations`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.podAnnotations) }} - - DEPRECATION WARNING: - `workers.podAnnotations` has been renamed to `workers.celery.podAnnotations`/`workers.kubernetes.podAnnotations`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.labels) }} - - DEPRECATION WARNING: - `workers.labels` has been renamed to `workers.celery.labels`/`workers.kubernetes.labels`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.volumeClaimTemplates) }} - - DEPRECATION WARNING: - `workers.volumeClaimTemplates` has been renamed to `workers.celery.volumeClaimTemplates`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.schedulerName) }} - - DEPRECATION WARNING: - `workers.schedulerName` has been renamed to `workers.celery.schedulerName`/`workers.kubernetes.schedulerName`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.logGroomerSidecar.enabled }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.enabled` has been renamed to `workers.celery.logGroomerSidecar.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.logGroomerSidecar.command) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.command` has been renamed to `workers.celery.logGroomerSidecar.command`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (.Values.workers.logGroomerSidecar.args | toJson) (list "bash" "/clean-logs" | toJson) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.args` has been renamed to `workers.celery.logGroomerSidecar.args`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.logGroomerSidecar.retentionDays) 15 }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.retentionDays` has been renamed to `workers.celery.logGroomerSidecar.retentionDays`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.logGroomerSidecar.retentionMinutes) 0 }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.retentionMinutes` has been renamed to `workers.celery.logGroomerSidecar.retentionMinutes`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.logGroomerSidecar.frequencyMinutes) 15 }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.frequencyMinutes` has been renamed to `workers.celery.logGroomerSidecar.frequencyMinutes`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.logGroomerSidecar.maxSizeBytes) 0 }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.maxSizeBytes` has been renamed to `workers.celery.logGroomerSidecar.maxSizeBytes`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if ne (int .Values.workers.logGroomerSidecar.maxSizePercent) 0 }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.maxSizePercent` has been renamed to `workers.celery.logGroomerSidecar.maxSizePercent`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.logGroomerSidecar.resources) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.resources` has been renamed to `workers.celery.logGroomerSidecar.resources`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.logGroomerSidecar.securityContexts.container) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.securityContexts.container` has been renamed to `workers.celery.logGroomerSidecar.securityContexts.container`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.logGroomerSidecar.env) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.env` has been renamed to `workers.celery.logGroomerSidecar.env`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.logGroomerSidecar.containerLifecycleHooks) }} - - DEPRECATION WARNING: - `workers.logGroomerSidecar.containerLifecycleHooks` has been renamed to `workers.celery.logGroomerSidecar.containerLifecycleHooks`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not .Values.workers.waitForMigrations.enabled }} - - DEPRECATION WARNING: - `workers.waitForMigrations.enabled` has been renamed to `workers.celery.waitForMigrations.enabled`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.waitForMigrations.env) }} - - DEPRECATION WARNING: - `workers.waitForMigrations.env` has been renamed to `workers.celery.waitForMigrations.env`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.waitForMigrations.securityContexts.container) }} - - DEPRECATION WARNING: - `workers.waitForMigrations.securityContexts.container` has been renamed to `workers.celery.waitForMigrations.securityContexts.container`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - -{{- if not (empty .Values.workers.env) }} - - DEPRECATION WARNING: - `workers.env` has been renamed to `workers.celery.env`/`workers.kubernetes.env`. - Please change your values as support for the old name will be dropped in a future release. - -{{- end }} - {{- if not .Values.dags.gitSync.recommendedProbeSetting }} DEPRECATION WARNING: diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index d2aa1e093e4e8..b05aac9b43535 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -84,13 +84,7 @@ If release name contains chart name it will be used as a full name. name: {{ template "airflow_metadata_secret" . }} key: connection {{- end }} - {{- $kedaEnabled := .Values.workers.keda.enabled }} - {{- $kedaUsePgBouncer := .Values.workers.keda.usePgbouncer }} - {{- if hasKey .Values.workers "celery" }} - {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} - {{- $kedaUsePgBouncer = or .Values.workers.celery.keda.usePgbouncer (and (not (has .Values.workers.celery.keda.usePgbouncer (list true false))) .Values.workers.keda.usePgbouncer) }} - {{- end }} - {{- if and $kedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not $kedaUsePgBouncer))) }} + {{- if and .Values.workers.celery.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer))) }} - name: KEDA_DB_CONN valueFrom: secretKeyRef: @@ -718,11 +712,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{/* Create the name of the worker service account to use */}} -{{- define "worker.serviceAccountName" -}} - {{- if and (hasKey .Values.workers "name") (ne .Values.workers.name "default") }} - {{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" (printf "%s-%s" "worker" .Values.workers.name)) .) -}} +{{- define "worker.celery.serviceAccountName" -}} + {{- if and (hasKey .Values.workers.celery "name") (ne .Values.workers.celery.name "default") }} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "celery" "nameSuffix" (printf "%s-%s" "worker" .Values.workers.celery.name)) .) -}} {{- else }} - {{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" "worker") .) -}} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "celery" "nameSuffix" "worker") .) -}} {{- end }} {{- end }} @@ -1094,8 +1088,6 @@ Usage: {{- define "workersMergeValues" -}} {{- $inputMap := index . 0 -}} {{- $overwriteMap := index . 1 -}} - {{- $sectionName := index . 2 -}} - {{- $orBoolean := index . 3 -}} {{- $outputMap := dict -}} {{- $fullOverwrite := list "annotations" "podAnnotations" "persistentVolumeClaimRetentionPolicy" "pod" "container" "securityContext" "containerLifecycleHooks" "config" "advanced" "behavior" "resources" "nodeSelector" "affinity" "labels" -}} @@ -1106,18 +1098,13 @@ Usage: {{- $_ := set $outputMap $key (get $overwriteMap $key) -}} {{/* Below if holds base logic for merge values */}} {{- else if and (hasKey $overwriteMap $key) (kindIs "map" $val) -}} - {{- $nested := include "workersMergeValues" (list $val (get $overwriteMap $key) $key $orBoolean) | fromYaml -}} + {{- $nested := include "workersMergeValues" (list $val (get $overwriteMap $key)) | fromYaml -}} {{- if gt (len $nested) 0 -}} {{- $_ := set $outputMap $key $nested -}} {{- end -}} {{/* Below if contains check for `slice` to fully overwrite list parameters */}} {{- else if and (hasKey $overwriteMap $key) (not (and (kindIs "slice" (get $overwriteMap $key)) (eq (len (get $overwriteMap $key)) 0))) -}} - {{/* Below if holds `or` logic for boolean fields in a map */}} - {{- if and (kindIs "bool" $val) (has $sectionName $orBoolean) -}} - {{- $_ := set $outputMap $key (or $val (get $overwriteMap $key)) -}} - {{- else -}} - {{- $_ := set $outputMap $key (get $overwriteMap $key) -}} - {{- end -}} + {{- $_ := set $outputMap $key (get $overwriteMap $key) -}} {{- else -}} {{- $_ := set $outputMap $key $val -}} {{- end -}} @@ -1130,24 +1117,3 @@ Usage: {{- end -}} {{- toYaml $outputMap -}} {{- end -}} - -{{/* -Remove all nil fields in a map and return new dict object. - -Usage: - {{ include "removeNilFields" . }} -*/}} -{{- define "removeNilFields" -}} - {{- $newValues := dict -}} - {{- range $key, $val := . -}} - {{- if kindIs "map" $val -}} - {{- $nested := include "removeNilFields" $val | fromYaml -}} - {{- if gt (len $nested) 0 -}} - {{- $_ := set $newValues $key $nested -}} - {{- end -}} - {{- else if not (eq $val nil) -}} - {{- $_ := set $newValues $key $val -}} - {{- end -}} - {{- end -}} - {{- toYaml $newValues -}} -{{- end -}} diff --git a/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml b/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml index 579571ca0d655..f1f83b93d4ec8 100644 --- a/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml +++ b/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml @@ -20,11 +20,7 @@ ########################## ## Pgbouncer NetworkPolicy ########################## -{{- $kedaEnabled := .Values.workers.keda.enabled }} -{{- if hasKey .Values.workers "celery" }} - {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} -{{- end }} -{{- $workersKedaEnabled := and $kedaEnabled (contains "CeleryExecutor" .Values.executor) }} +{{- $workersKedaEnabled := and .Values.workers.celery.keda.enabled (contains "CeleryExecutor" .Values.executor) }} {{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} {{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 @@ -55,9 +51,9 @@ spec: tier: airflow release: {{ .Release.Name }} {{- if or $workersKedaEnabled $triggererKedaEnabled }} - {{- if and $workersKedaEnabled (.Values.workers.celery.keda.namespaceLabels | default .Values.workers.keda.namespaceLabels) }} + {{- if and $workersKedaEnabled .Values.workers.celery.keda.namespaceLabels }} - namespaceSelector: - matchLabels: {{- toYaml (.Values.workers.celery.keda.namespaceLabels | default .Values.workers.keda.namespaceLabels) | nindent 10 }} + matchLabels: {{- toYaml .Values.workers.celery.keda.namespaceLabels | nindent 10 }} podSelector: {{- else if and .Values.triggerer.enabled .Values.triggerer.keda.namespaceLabels }} - namespaceSelector: diff --git a/chart/templates/rbac/job-launcher-rolebinding.yaml b/chart/templates/rbac/job-launcher-rolebinding.yaml index e5cbca8c630b4..750192be750bc 100644 --- a/chart/templates/rbac/job-launcher-rolebinding.yaml +++ b/chart/templates/rbac/job-launcher-rolebinding.yaml @@ -71,7 +71,7 @@ subjects: {{- $executorClass := last (splitList "." ($executor | trim)) }} {{- if has $executorClass $workerLaunchExecutors }} - kind: ServiceAccount - name: {{ include "worker.serviceAccountName" $ }} + name: {{ include "worker.celery.serviceAccountName" $ }} namespace: "{{ $.Release.Namespace }}" {{- break }} {{- end }} diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml index a9a3b510a4379..b08ff79493dd0 100644 --- a/chart/templates/rbac/pod-launcher-rolebinding.yaml +++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml @@ -71,7 +71,7 @@ subjects: {{- $executorClass := last (splitList "." ($executor | trim)) }} {{- if has $executorClass $workerLaunchExecutors }} - kind: ServiceAccount - name: {{ include "worker.serviceAccountName" $ }} + name: {{ include "worker.celery.serviceAccountName" $ }} namespace: "{{ $.Release.Namespace }}" {{- break }} {{- end }} diff --git a/chart/templates/rbac/security-context-constraint-rolebinding.yaml b/chart/templates/rbac/security-context-constraint-rolebinding.yaml index ecb1aff506d58..fe34395753f75 100644 --- a/chart/templates/rbac/security-context-constraint-rolebinding.yaml +++ b/chart/templates/rbac/security-context-constraint-rolebinding.yaml @@ -50,9 +50,14 @@ roleRef: kind: ClusterRole name: system:openshift:scc:anyuid subjects: - {{- if or (contains "CeleryExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) }} + {{- if contains "CeleryExecutor" .Values.executor }} - kind: ServiceAccount - name: {{ include "worker.serviceAccountName" . }} + name: {{ include "worker.celery.serviceAccountName" . }} + namespace: "{{ .Release.Namespace }}" + {{- end }} + {{- if contains "KubernetesExecutor" .Values.executor }} + - kind: ServiceAccount + name: {{ include "worker.kubernetes.serviceAccountName" . }} namespace: "{{ .Release.Namespace }}" {{- end }} {{- if .Values.scheduler.enabled }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 3ba9d89dae495..2c64e29da576c 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -22,10 +22,8 @@ ########################################### {{- if .Values.scheduler.enabled }} {{- $local := contains "Local" .Values.executor }} -# Is persistence enabled on the _workers_? # This is important because in $local mode, the scheduler assumes the role of the worker -{{- $persistence := or .Values.workers.celery.persistence.enabled (and (not (has .Values.workers.celery.persistence.enabled (list true false))) .Values.workers.persistence.enabled) }} -{{- $stateful := and $local $persistence }} +{{- $stateful := and $local .Values.workers.celery.persistence.enabled }} # We can skip Dags mounts on scheduler if dagProcessor is enabled, except with $local mode {{- $localOrDagProcessorDisabled := or (not .Values.dagProcessor.enabled) $local }} {{- $remoteLogging := or .Values.elasticsearch.enabled .Values.opensearch.enabled }} @@ -68,8 +66,8 @@ spec: {{- if and $stateful .Values.scheduler.updateStrategy }} updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }} {{- end }} - {{- if and $stateful (or .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) }} - persistentVolumeClaimRetentionPolicy: {{- toYaml (.Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy | default .Values.workers.persistence.persistentVolumeClaimRetentionPolicy) | nindent 4 }} + {{- if and $stateful .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} {{- end }} {{- if and (not $stateful) .Values.scheduler.strategy }} strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }} @@ -370,16 +368,16 @@ spec: kind: PersistentVolumeClaim metadata: name: logs - {{- if or .Values.workers.celery.persistence.annotations .Values.workers.persistence.annotations }} - annotations: {{- toYaml (.Values.workers.celery.persistence.annotations | default .Values.workers.persistence.annotations) | nindent 10 }} + {{- if .Values.workers.celery.persistence.annotations }} + annotations: {{- toYaml .Values.workers.celery.persistence.annotations | nindent 10 }} {{- end }} spec: - {{- if or .Values.workers.celery.persistence.storageClassName .Values.workers.persistence.storageClassName }} - storageClassName: {{ tpl (.Values.workers.celery.persistence.storageClassName | default .Values.workers.persistence.storageClassName) . | quote }} + {{- if .Values.workers.celery.persistence.storageClassName }} + storageClassName: {{ tpl .Values.workers.celery.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: requests: - storage: {{ .Values.workers.celery.persistence.size | default .Values.workers.persistence.size }} + storage: {{ .Values.workers.celery.persistence.size }} {{- end }} {{- end }} diff --git a/chart/templates/secrets/metadata-connection-secret.yaml b/chart/templates/secrets/metadata-connection-secret.yaml index 71441755d4bae..66757ec03c650 100644 --- a/chart/templates/secrets/metadata-connection-secret.yaml +++ b/chart/templates/secrets/metadata-connection-secret.yaml @@ -31,12 +31,6 @@ {{- $metadataDatabase := tpl .Values.data.metadataConnection.db . }} {{- $database := ternary (printf "%s-%s" .Release.Name "metadata") $metadataDatabase .Values.pgbouncer.enabled }} {{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }} -{{- $kedaEnabled := .Values.workers.keda.enabled }} -{{- $kedaUsePgBouncer := .Values.workers.keda.usePgbouncer }} -{{- if hasKey .Values.workers "celery" }} - {{- $kedaEnabled = or .Values.workers.celery.keda.enabled (and (not (has .Values.workers.celery.keda.enabled (list true false))) .Values.workers.keda.enabled) }} - {{- $kedaUsePgBouncer = or .Values.workers.celery.keda.usePgbouncer (and (not (has .Values.workers.celery.keda.usePgbouncer (list true false))) .Values.workers.keda.usePgbouncer) }} -{{- end }} apiVersion: v1 kind: Secret metadata: @@ -58,11 +52,11 @@ data: {{- with .Values.data.metadataConnection }} connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} {{- end }} - {{- if and $kedaEnabled .Values.pgbouncer.enabled (not $kedaUsePgBouncer) }} + {{- if and .Values.workers.celery.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer) }} {{- with .Values.data.metadataConnection }} kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} {{- end }} - {{- else if and (or $kedaEnabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }} + {{- else if and (or .Values.workers.celery.keda.enabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }} {{- with .Values.data.metadataConnection }} kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index e3fb42478d768..f3addaa9e4520 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -21,45 +21,42 @@ ## Airflow Worker Deployment ############################ {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} {{- if contains "CeleryExecutor" .Values.executor }} --- -{{- $persistence := or .Values.workers.persistence.enabled }} -{{- $keda := .Values.workers.keda.enabled }} -{{- $hpa := and .Values.workers.hpa.enabled (not .Values.workers.keda.enabled) }} -{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }} -{{- $affinity := or .Values.workers.affinity .Values.affinity }} -{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }} -{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} -{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.workers.revisionHistoryLimit .Values.revisionHistoryLimit) }} -{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers .Values) }} -{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers .Values) }} -{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list .Values.workers.persistence .Values) }} -{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.workers.waitForMigrations .Values) }} -{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list .Values.workers.logGroomerSidecar .Values) }} -{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kerberosSidecar .Values) }} -{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.kerberosInitContainer .Values) }} -{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} -{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (.Values.workers.safeToEvict | toString) }} -{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.podAnnotations }} -{{- $schedulerName := or .Values.workers.schedulerName .Values.schedulerName }} +{{- $persistence := .Values.workers.celery.persistence.enabled }} +{{- $keda := .Values.workers.celery.keda.enabled }} +{{- $hpa := and .Values.workers.celery.hpa.enabled (not .Values.workers.celery.keda.enabled) }} +{{- $nodeSelector := or .Values.workers.celery.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.workers.celery.affinity .Values.affinity }} +{{- $tolerations := or .Values.workers.celery.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.workers.celery.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.workers.celery.revisionHistoryLimit .Values.revisionHistoryLimit) }} +{{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers.celery .Values) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers.celery .Values) }} +{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list .Values.workers.celery.persistence .Values) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.workers.celery.waitForMigrations .Values) }} +{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list .Values.workers.celery.logGroomerSidecar .Values) }} +{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.celery.kerberosSidecar .Values) }} +{{- $containerSecurityContextKerberosInitContainer := include "containerSecurityContext" (list .Values.workers.celery.kerberosInitContainer .Values) }} +{{- $containerLifecycleHooksKerberosInitContainer := or .Values.workers.celery.kerberosInitContainer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooks := or .Values.workers.celery.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.celery.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.celery.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (.Values.workers.celery.safeToEvict | toString) }} +{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.celery.podAnnotations }} +{{- $schedulerName := or .Values.workers.celery.schedulerName .Values.schedulerName }} apiVersion: apps/v1 kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }} metadata: - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: worker @@ -69,38 +66,38 @@ metadata: {{- with .Values.labels }} {{- toYaml . | nindent 4 }} {{- end }} - {{- if .Values.workers.annotations }} - annotations: {{- toYaml .Values.workers.annotations | nindent 4 }} + {{- if .Values.workers.celery.annotations }} + annotations: {{- toYaml .Values.workers.celery.annotations | nindent 4 }} {{- end }} spec: {{- if $persistence }} - serviceName: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + serviceName: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} {{- end }} {{- if and (not $keda) (not $hpa) }} - replicas: {{ .Values.workers.replicas }} + replicas: {{ .Values.workers.celery.replicas }} {{- end }} {{- if ne $revisionHistoryLimit "" }} revisionHistoryLimit: {{ $revisionHistoryLimit }} {{- end }} - {{- if and $persistence .Values.workers.persistence.persistentVolumeClaimRetentionPolicy }} - persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} + {{- if and $persistence .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.workers.celery.persistence.persistentVolumeClaimRetentionPolicy | nindent 4 }} {{- end }} selector: matchLabels: tier: airflow component: worker release: {{ .Release.Name }} - {{- if ne .Values.workers.name "default" }} - worker-set: {{ .Values.workers.name }} + {{- if ne .Values.workers.celery.name "default" }} + worker-set: {{ .Values.workers.celery.name }} {{- end }} - {{- if and $persistence .Values.workers.podManagementPolicy }} - podManagementPolicy: {{ .Values.workers.podManagementPolicy }} + {{- if and $persistence .Values.workers.celery.podManagementPolicy }} + podManagementPolicy: {{ .Values.workers.celery.podManagementPolicy }} {{- end }} - {{- if and $persistence .Values.workers.updateStrategy }} - updateStrategy: {{- toYaml .Values.workers.updateStrategy | nindent 4 }} + {{- if and $persistence .Values.workers.celery.updateStrategy }} + updateStrategy: {{- toYaml .Values.workers.celery.updateStrategy | nindent 4 }} {{- end }} - {{- if and (not $persistence) .Values.workers.strategy }} - strategy: {{- toYaml .Values.workers.strategy | nindent 4 }} + {{- if and (not $persistence) .Values.workers.celery.strategy }} + strategy: {{- toYaml .Values.workers.celery.strategy | nindent 4 }} {{- end }} template: metadata: @@ -108,11 +105,11 @@ spec: tier: airflow component: worker release: {{ .Release.Name }} - {{- if ne .Values.workers.name "default" }} - worker-set: {{ .Values.workers.name }} + {{- if ne .Values.workers.celery.name "default" }} + worker-set: {{ .Values.workers.celery.name }} {{- end }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 8 }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 8 }} {{- end }} annotations: checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} @@ -126,11 +123,11 @@ spec: {{- tpl (toYaml $podAnnotations) . | nindent 8 }} {{- end }} spec: - {{- if .Values.workers.runtimeClassName }} - runtimeClassName: {{ .Values.workers.runtimeClassName }} + {{- if .Values.workers.celery.runtimeClassName }} + runtimeClassName: {{ .Values.workers.celery.runtimeClassName }} {{- end }} - {{- if .Values.workers.priorityClassName }} - priorityClassName: {{ .Values.workers.priorityClassName }} + {{- if .Values.workers.celery.priorityClassName }} + priorityClassName: {{ .Values.workers.celery.priorityClassName }} {{- end }} {{- if $schedulerName }} schedulerName: {{ $schedulerName }} @@ -151,24 +148,24 @@ spec: {{- end }} tolerations: {{- toYaml $tolerations | nindent 8 }} topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} - {{- if .Values.workers.hostAliases }} - hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 8 }} + {{- if .Values.workers.celery.hostAliases }} + hostAliases: {{- toYaml .Values.workers.celery.hostAliases | nindent 8 }} {{- end }} - terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.workers.celery.terminationGracePeriodSeconds }} restartPolicy: Always - serviceAccountName: {{ include "worker.serviceAccountName" . }} + serviceAccountName: {{ include "worker.celery.serviceAccountName" . }} securityContext: {{ $securityContext | nindent 8 }} imagePullSecrets: {{ include "image_pull_secrets" . | nindent 8 }} initContainers: - {{- if and $persistence .Values.workers.persistence.fixPermissions }} + {{- if and $persistence .Values.workers.celery.persistence.fixPermissions }} - name: volume-permissions - resources: {{- toYaml .Values.workers.resources | nindent 12 }} + resources: {{- toYaml .Values.workers.celery.resources | nindent 12 }} image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} command: - chown - -R - - "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers) }}" + - "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers.celery) }}" - {{ template "airflow_logs" . }} securityContext: {{ $containerSecurityContextPersistence | nindent 12 }} volumeMounts: @@ -178,7 +175,7 @@ spec: subPath: {{ .Values.logs.persistence.subPath }} {{- end }} {{- end }} - {{- if .Values.workers.kerberosInitContainer.enabled }} + {{- if .Values.workers.celery.kerberosInitContainer.enabled }} - name: kerberos-init image: {{ template "airflow_image" . }} securityContext: {{ $containerSecurityContextKerberosInitContainer | nindent 12 }} @@ -187,7 +184,7 @@ spec: {{- end }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} args: ["kerberos", "-o"] - resources: {{- toYaml .Values.workers.kerberosInitContainer.resources | nindent 12 }} + resources: {{- toYaml .Values.workers.celery.kerberosInitContainer.resources | nindent 12 }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} @@ -209,8 +206,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- if .Values.workers.celery.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.celery.extraVolumeMounts) . | nindent 12 }} {{- end }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: @@ -221,9 +218,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- end }} - {{- if .Values.workers.waitForMigrations.enabled }} + {{- if .Values.workers.celery.waitForMigrations.enabled }} - name: wait-for-airflow-migrations - resources: {{- toYaml .Values.workers.resources | nindent 12 }} + resources: {{- toYaml .Values.workers.celery.resources | nindent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} @@ -237,23 +234,23 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- if .Values.workers.celery.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.celery.extraVolumeMounts) . | nindent 12 }} {{- end }} args: {{- include "wait-for-migrations-command" . | indent 10 }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} - {{- if .Values.workers.waitForMigrations.env }} - {{- tpl (toYaml .Values.workers.waitForMigrations.env) $ | nindent 12 }} + {{- if .Values.workers.celery.waitForMigrations.env }} + {{- tpl (toYaml .Values.workers.celery.waitForMigrations.env) $ | nindent 12 }} {{- end }} {{- end }} {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} {{- end }} - {{- if .Values.workers.extraInitContainers }} - {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 8 }} + {{- if .Values.workers.celery.extraInitContainers }} + {{- tpl (toYaml .Values.workers.celery.extraInitContainers) . | nindent 8 }} {{- end }} containers: - name: worker @@ -263,23 +260,23 @@ spec: {{- if $containerLifecycleHooks }} lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} {{- end }} - {{- if .Values.workers.command }} - command: {{ tpl (toYaml .Values.workers.command) . | nindent 12 }} + {{- if .Values.workers.celery.command }} + command: {{ tpl (toYaml .Values.workers.celery.command) . | nindent 12 }} {{- end }} - {{- if .Values.workers.args }} - args: {{ tpl (toYaml .Values.workers.args) . | nindent 12 }} + {{- if .Values.workers.celery.args }} + args: {{ tpl (toYaml .Values.workers.celery.args) . | nindent 12 }} {{- end }} - resources: {{- toYaml .Values.workers.resources | nindent 12 }} - {{- if .Values.workers.livenessProbe.enabled }} + resources: {{- toYaml .Values.workers.celery.resources | nindent 12 }} + {{- if .Values.workers.celery.livenessProbe.enabled }} livenessProbe: - initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }} - timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }} - periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }} + initialDelaySeconds: {{ .Values.workers.celery.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.workers.celery.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.workers.celery.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.workers.celery.livenessProbe.periodSeconds }} exec: command: - {{- if .Values.workers.livenessProbe.command }} - {{- toYaml .Values.workers.livenessProbe.command | nindent 16 }} + {{- if .Values.workers.celery.livenessProbe.command }} + {{- toYaml .Values.workers.celery.livenessProbe.command | nindent 16 }} {{- else }} - sh - -c @@ -287,8 +284,8 @@ spec: {{- end }} {{- end }} ports: - {{- if .Values.workers.extraPorts }} - {{- toYaml .Values.workers.extraPorts | nindent 12 }} + {{- if .Values.workers.celery.extraPorts }} + {{- toYaml .Values.workers.celery.extraPorts | nindent 12 }} {{- end }} - name: worker-logs containerPort: {{ .Values.ports.workerLogs }} @@ -296,8 +293,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- if .Values.workers.celery.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.celery.extraVolumeMounts) . | nindent 12 }} {{- end }} - name: logs mountPath: {{ template "airflow_logs" . }} @@ -328,8 +325,8 @@ spec: value: "0" {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} - {{- include "container_extra_envs" (list . .Values.workers.env) | indent 10 }} - {{- if .Values.workers.kerberosSidecar.enabled }} + {{- include "container_extra_envs" (list . .Values.workers.celery.env) | indent 10 }} + {{- if .Values.workers.celery.kerberosSidecar.enabled }} - name: KRB5_CONFIG value: {{ .Values.kerberos.configPath | quote }} - name: KRB5CCNAME @@ -338,7 +335,7 @@ spec: {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" . | nindent 8 }} {{- end }} - {{- if .Values.workers.logGroomerSidecar.enabled }} + {{- if .Values.workers.celery.logGroomerSidecar.enabled }} - name: worker-log-groomer image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -346,39 +343,39 @@ spec: {{- if $containerLifecycleHooksLogGroomerSidecar }} lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }} {{- end }} - {{- if .Values.workers.logGroomerSidecar.command }} - command: {{ tpl (toYaml .Values.workers.logGroomerSidecar.command) . | nindent 12 }} + {{- if .Values.workers.celery.logGroomerSidecar.command }} + command: {{ tpl (toYaml .Values.workers.celery.logGroomerSidecar.command) . | nindent 12 }} {{- end }} - {{- if .Values.workers.logGroomerSidecar.args }} - args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }} + {{- if .Values.workers.celery.logGroomerSidecar.args }} + args: {{ tpl (toYaml .Values.workers.celery.logGroomerSidecar.args) . | nindent 12 }} {{- end }} env: - {{- if ne .Values.workers.logGroomerSidecar.retentionDays nil }} + {{- if ne .Values.workers.celery.logGroomerSidecar.retentionDays nil }} - name: AIRFLOW__LOG_RETENTION_DAYS - value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}" + value: "{{ .Values.workers.celery.logGroomerSidecar.retentionDays }}" {{- end }} - {{- if ne .Values.workers.logGroomerSidecar.retentionMinutes nil }} + {{- if ne .Values.workers.celery.logGroomerSidecar.retentionMinutes nil }} - name: AIRFLOW__LOG_RETENTION_MINUTES - value: "{{ .Values.workers.logGroomerSidecar.retentionMinutes }}" + value: "{{ .Values.workers.celery.logGroomerSidecar.retentionMinutes }}" {{- end }} - {{- if .Values.workers.logGroomerSidecar.frequencyMinutes }} + {{- if .Values.workers.celery.logGroomerSidecar.frequencyMinutes }} - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES - value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}" + value: "{{ .Values.workers.celery.logGroomerSidecar.frequencyMinutes }}" {{- end }} - {{- if .Values.workers.logGroomerSidecar.maxSizeBytes }} + {{- if .Values.workers.celery.logGroomerSidecar.maxSizeBytes }} - name: AIRFLOW__LOG_MAX_SIZE_BYTES - value: "{{ .Values.workers.logGroomerSidecar.maxSizeBytes | int64 }}" + value: "{{ .Values.workers.celery.logGroomerSidecar.maxSizeBytes | int64 }}" {{- end }} - {{- if .Values.workers.logGroomerSidecar.maxSizePercent }} + {{- if .Values.workers.celery.logGroomerSidecar.maxSizePercent }} - name: AIRFLOW__LOG_MAX_SIZE_PERCENT - value: "{{ .Values.workers.logGroomerSidecar.maxSizePercent }}" + value: "{{ .Values.workers.celery.logGroomerSidecar.maxSizePercent }}" {{- end }} - name: AIRFLOW_HOME value: "{{ .Values.airflowHome }}" - {{- if .Values.workers.logGroomerSidecar.env }} - {{- tpl (toYaml .Values.workers.logGroomerSidecar.env) $ | nindent 12 }} + {{- if .Values.workers.celery.logGroomerSidecar.env }} + {{- tpl (toYaml .Values.workers.celery.logGroomerSidecar.env) $ | nindent 12 }} {{- end }} - resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }} + resources: {{- toYaml .Values.workers.celery.logGroomerSidecar.resources | nindent 12 }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} @@ -388,11 +385,11 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- if .Values.workers.celery.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.celery.extraVolumeMounts) . | nindent 12 }} {{- end }} {{- end }} - {{- if .Values.workers.kerberosSidecar.enabled }} + {{- if .Values.workers.celery.kerberosSidecar.enabled }} - name: worker-kerberos image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -401,7 +398,7 @@ spec: lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 12 }} {{- end }} args: ["kerberos"] - resources: {{- toYaml .Values.workers.kerberosSidecar.resources | nindent 12 }} + resources: {{- toYaml .Values.workers.celery.kerberosSidecar.resources | nindent 12 }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} @@ -423,8 +420,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }} + {{- if .Values.workers.celery.extraVolumeMounts }} + {{- tpl (toYaml .Values.workers.celery.extraVolumeMounts) . | nindent 12 }} {{- end }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: @@ -435,15 +432,15 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- end }} - {{- if .Values.workers.extraContainers }} - {{- tpl (toYaml .Values.workers.extraContainers) . | nindent 8 }} + {{- if .Values.workers.celery.extraContainers }} + {{- tpl (toYaml .Values.workers.celery.extraContainers) . | nindent 8 }} {{- end }} volumes: {{- if .Values.volumes }} {{- toYaml .Values.volumes | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumes }} - {{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 8 }} + {{- if .Values.workers.celery.extraVolumes }} + {{- tpl (toYaml .Values.workers.celery.extraVolumes) . | nindent 8 }} {{- end }} - name: config configMap: @@ -474,26 +471,26 @@ spec: - name: logs emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }} {{- end }} - {{- if and $persistence (or (not .Values.logs.persistence.enabled) .Values.workers.volumeClaimTemplates) }} + {{- if and $persistence (or (not .Values.logs.persistence.enabled) .Values.workers.celery.volumeClaimTemplates) }} volumeClaimTemplates: {{- if not .Values.logs.persistence.enabled }} - apiVersion: v1 kind: PersistentVolumeClaim metadata: name: logs - {{- if .Values.workers.persistence.annotations }} - annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }} + {{- if .Values.workers.celery.persistence.annotations }} + annotations: {{- toYaml .Values.workers.celery.persistence.annotations | nindent 10 }} {{- end }} spec: - {{- if .Values.workers.persistence.storageClassName }} - storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }} + {{- if .Values.workers.celery.persistence.storageClassName }} + storageClassName: {{ tpl .Values.workers.celery.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: requests: - storage: {{ .Values.workers.persistence.size }} + storage: {{ .Values.workers.celery.persistence.size }} {{- end }} - {{- with .Values.workers.volumeClaimTemplates }} + {{- with .Values.workers.celery.volumeClaimTemplates }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-hpa.yaml b/chart/templates/workers/worker-hpa.yaml index 847488a9ea48a..c77ebb9dd88d0 100644 --- a/chart/templates/workers/worker-hpa.yaml +++ b/chart/templates/workers/worker-hpa.yaml @@ -21,42 +21,39 @@ ## Airflow Worker HPA ##################### {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} -{{- if and (and (not .Values.workers.keda.enabled) .Values.workers.hpa.enabled) (contains "CeleryExecutor" .Values.executor) }} +{{- if and (and (not .Values.workers.celery.keda.enabled) .Values.workers.celery.hpa.enabled) (contains "CeleryExecutor" .Values.executor) }} --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: worker-horizontalpodautoscaler release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: scaleTargetRef: apiVersion: apps/v1 - kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }} - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} - minReplicas: {{ .Values.workers.hpa.minReplicaCount }} - maxReplicas: {{ .Values.workers.hpa.maxReplicaCount }} - metrics: {{- toYaml .Values.workers.hpa.metrics | nindent 4 }} - {{- with .Values.workers.hpa.behavior }} + kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.celery.persistence.enabled }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} + minReplicas: {{ .Values.workers.celery.hpa.minReplicaCount }} + maxReplicas: {{ .Values.workers.celery.hpa.maxReplicaCount }} + metrics: {{- toYaml .Values.workers.celery.hpa.metrics | nindent 4 }} + {{- with .Values.workers.celery.hpa.behavior }} behavior: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml index 64d4474692718..50ee1c92b7dd9 100644 --- a/chart/templates/workers/worker-kedaautoscaler.yaml +++ b/chart/templates/workers/worker-kedaautoscaler.yaml @@ -21,44 +21,41 @@ ## Airflow Worker KEDA Scaler ############################# {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} -{{- if and .Values.workers.keda.enabled (contains "CeleryExecutor" .Values.executor) }} +{{- if and .Values.workers.celery.keda.enabled (contains "CeleryExecutor" .Values.executor) }} --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: worker-horizontalpodautoscaler release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + deploymentName: {{ .Release.Name }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: scaleTargetRef: - kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }} - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.celery.persistence.enabled }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} envSourceContainerName: worker - pollingInterval: {{ .Values.workers.keda.pollingInterval }} - cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }} - minReplicaCount: {{ .Values.workers.keda.minReplicaCount }} - maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }} - {{- if .Values.workers.keda.advanced }} - advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }} + pollingInterval: {{ .Values.workers.celery.keda.pollingInterval }} + cooldownPeriod: {{ .Values.workers.celery.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.workers.celery.keda.minReplicaCount }} + maxReplicaCount: {{ .Values.workers.celery.keda.maxReplicaCount }} + {{- if .Values.workers.celery.keda.advanced }} + advanced: {{- toYaml .Values.workers.celery.keda.advanced | nindent 4 }} {{- end }} triggers: {{- if eq .Values.data.metadataConnection.protocol "mysql" }} @@ -66,17 +63,17 @@ spec: metadata: queryValue: "1" connectionStringFromEnv: KEDA_DB_CONN - query: {{ tpl .Values.workers.keda.query . | quote }} + query: {{ tpl .Values.workers.celery.keda.query . | quote }} {{- else }} - type: "postgresql" metadata: targetQueryValue: "1" - {{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }} + {{- if and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer) }} connectionFromEnv: KEDA_DB_CONN {{- else }} connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB {{- end }} - query: {{ tpl .Values.workers.keda.query . | quote }} + query: {{ tpl .Values.workers.celery.keda.query . | quote }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml index 82cf72f6b6801..1ab72ec9789a0 100644 --- a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml +++ b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml @@ -23,7 +23,7 @@ {{- if and .Values.workers.kubernetes.serviceAccount.create (contains "KubernetesExecutor" .Values.executor) }} apiVersion: v1 kind: ServiceAccount -automountServiceAccountToken: {{ or .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (and (not (has .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (list true false))) .Values.workers.serviceAccount.automountServiceAccountToken) }} +automountServiceAccountToken: {{ .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken }} metadata: name: {{ include "worker.kubernetes.serviceAccountName" . }} labels: @@ -32,10 +32,10 @@ metadata: release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} - {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.kubernetes.labels }} + {{- mustMerge .Values.workers.kubernetes.labels .Values.labels | toYaml | nindent 4 }} {{- end }} - {{- with (.Values.workers.kubernetes.serviceAccount.annotations | default .Values.workers.serviceAccount.annotations) }} + {{- with .Values.workers.kubernetes.serviceAccount.annotations }} annotations: {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} diff --git a/chart/templates/workers/worker-networkpolicy.yaml b/chart/templates/workers/worker-networkpolicy.yaml index ee0f6a26d1a9a..a3ff99272f3dc 100644 --- a/chart/templates/workers/worker-networkpolicy.yaml +++ b/chart/templates/workers/worker-networkpolicy.yaml @@ -21,31 +21,28 @@ ## Airflow Worker NetworkPolicy ############################### {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} {{- if and .Values.networkPolicies.enabled (contains "CeleryExecutor" .Values.executor) }} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "airflow.fullname" . }}-worker-policy{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker-policy{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: airflow-worker-policy release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: podSelector: @@ -53,8 +50,8 @@ spec: tier: airflow component: worker release: {{ .Release.Name }} - {{- if ne .Values.workers.name "default" }} - worker-set: {{ .Values.workers.name }} + {{- if ne .Values.workers.celery.name "default" }} + worker-set: {{ .Values.workers.celery.name }} {{- end }} policyTypes: - Ingress diff --git a/chart/templates/workers/worker-poddisruptionbudget.yaml b/chart/templates/workers/worker-poddisruptionbudget.yaml index 8aed138c3be18..c31c93f967985 100644 --- a/chart/templates/workers/worker-poddisruptionbudget.yaml +++ b/chart/templates/workers/worker-poddisruptionbudget.yaml @@ -21,31 +21,28 @@ ## Airflow Worker PodDisruptionBudget ################################# {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} -{{- if .Values.workers.podDisruptionBudget.enabled }} +{{- if .Values.workers.celery.podDisruptionBudget.enabled }} --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: {{ include "airflow.fullname" . }}-worker-pdb{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker-pdb{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: worker release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: selector: @@ -53,10 +50,10 @@ spec: tier: airflow component: worker release: {{ .Release.Name }} - {{- if ne .Values.workers.name "default" }} - worker-set: {{ .Values.workers.name }} + {{- if ne .Values.workers.celery.name "default" }} + worker-set: {{ .Values.workers.celery.name }} {{- end }} - {{- toYaml .Values.workers.podDisruptionBudget.config | nindent 2 }} + {{- toYaml .Values.workers.celery.podDisruptionBudget.config | nindent 2 }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/workers/worker-service.yaml index 45a38d8b7934e..f6c9887c7f81d 100644 --- a/chart/templates/workers/worker-service.yaml +++ b/chart/templates/workers/worker-service.yaml @@ -21,31 +21,28 @@ ## Airflow Worker Service ######################### {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} {{- if contains "CeleryExecutor" .Values.executor }} --- apiVersion: v1 kind: Service metadata: - name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.name "default" }}-{{ .Values.workers.name }}{{ end }} + name: {{ include "airflow.fullname" . }}-worker{{ if ne .Values.workers.celery.name "default" }}-{{ .Values.workers.celery.name }}{{ end }} labels: tier: airflow component: worker release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: clusterIP: None @@ -53,8 +50,8 @@ spec: tier: airflow component: worker release: {{ .Release.Name }} - {{- if ne .Values.workers.name "default" }} - worker-set: {{ .Values.workers.name }} + {{- if ne .Values.workers.celery.name "default" }} + worker-set: {{ .Values.workers.celery.name }} {{- end }} ports: - name: worker-logs diff --git a/chart/templates/workers/worker-serviceaccount.yaml b/chart/templates/workers/worker-serviceaccount.yaml index cbcf95381e834..866e1d088d68e 100644 --- a/chart/templates/workers/worker-serviceaccount.yaml +++ b/chart/templates/workers/worker-serviceaccount.yaml @@ -21,34 +21,31 @@ ## Airflow Worker ServiceAccount ################################# {{- $globals := deepCopy . -}} -{{- $filteredCelery := include "removeNilFields" .Values.workers.celery | fromYaml -}} -{{- $mergedWorkers := (include "workersMergeValues" (list .Values.workers $filteredCelery "" (list "kerberosInitContainer" "kerberosSidecar")) | fromYaml) -}} -{{- $_ := unset $mergedWorkers "celery" -}} {{- $workerSets := .Values.workers.celery.sets | default list -}} {{- if .Values.workers.celery.enableDefault -}} {{- $workerSets = concat (list (dict "name" "default")) $workerSets -}} {{- end -}} {{- range $workerSet := $workerSets -}} - {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} - {{- $_ := set $globals.Values "workers" $workers -}} + {{- $workers := (include "workersMergeValues" (list $.Values.workers.celery $workerSet) | fromYaml) -}} + {{- $_ := set $globals.Values.workers "celery" $workers -}} {{- with $globals -}} -{{- if and .Values.workers.serviceAccount.create (include "airflow.podLaunchingExecutor" .) }} +{{- if and .Values.workers.celery.serviceAccount.create (include "airflow.podLaunchingExecutor" .) }} --- apiVersion: v1 kind: ServiceAccount -automountServiceAccountToken: {{ .Values.workers.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.workers.celery.serviceAccount.automountServiceAccountToken }} metadata: - name: {{ include "worker.serviceAccountName" . }} + name: {{ include "worker.celery.serviceAccountName" . }} labels: tier: airflow component: worker release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or .Values.labels .Values.workers.labels }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.celery.labels }} + {{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml | nindent 4 }} {{- end }} - {{- with .Values.workers.serviceAccount.annotations}} + {{- with .Values.workers.celery.serviceAccount.annotations}} annotations: {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} diff --git a/chart/tests/chart_utils/log_groomer.py b/chart/tests/chart_utils/log_groomer.py index 513d50c7ecf8a..d60cb323630cc 100644 --- a/chart/tests/chart_utils/log_groomer.py +++ b/chart/tests/chart_utils/log_groomer.py @@ -85,11 +85,9 @@ def test_log_groomer_collector_custom_env(self): if self.obj_name == "dag-processor": values = {"dagProcessor": {"logGroomerSidecar": {"env": env}}} - elif self.obj_name == "workers-celery": - values = {"workers": {"celery": {"logGroomerSidecar": {"env": env}}}} else: values = { - "workers": {"logGroomerSidecar": {"env": env}}, + "workers": {"celery": {"logGroomerSidecar": {"env": env}}}, "scheduler": {"logGroomerSidecar": {"env": env}}, "triggerer": {"logGroomerSidecar": {"env": env}}, } @@ -116,14 +114,7 @@ def test_log_groomer_command_and_args_overrides(self, command, args): docs = render_chart(values=values, show_only=self.get_show_only()) assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0]) - - if self.obj_name == "workers-celery" and args is None: - assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == [ - "bash", - "/clean-logs", - ] - else: - assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) def test_log_groomer_command_and_args_overrides_are_templated(self): if self.obj_name == "dag-processor": @@ -180,15 +171,6 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re ) == retention_result ) - elif self.obj_name == "workers-celery" and retention_result is None: - # Testing backward compatibility of move from workers to workers.celery - assert ( - jmespath.search( - "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", - docs[0], - ) - == "15" - ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 3 @@ -211,15 +193,6 @@ def test_log_groomer_retention_minutes_overrides(self, retention_minutes, retent ) == retention_result ) - elif self.obj_name == "workers-celery" and retention_result is None: - # Testing backward compatibility of move from workers to workers.celery - assert ( - jmespath.search( - "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_MINUTES'].value | [0]", - docs[0], - ) - == "0" - ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 3 @@ -242,15 +215,6 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque ) == frequency_result ) - elif self.obj_name == "workers-celery" and frequency_result is None: - # Testing backward compatibility of move from workers to workers.celery - assert ( - jmespath.search( - "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", - docs[0], - ) - == "15" - ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 3 diff --git a/chart/tests/helm_tests/airflow_aux/test_airflow_common.py b/chart/tests/helm_tests/airflow_aux/test_airflow_common.py index a46ea30bee25b..cafc8a3f41301 100644 --- a/chart/tests/helm_tests/airflow_aux/test_airflow_common.py +++ b/chart/tests/helm_tests/airflow_aux/test_airflow_common.py @@ -427,15 +427,7 @@ def test_have_all_config_mounts_on_init_containers(self): for doc in docs: assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", doc) - @pytest.mark.parametrize( - "workers_values", - [ - {"priorityClassName": "low-priority-worker"}, - {"celery": {"priorityClassName": "low-priority-worker"}}, - {"priorityClassName": "test", "celery": {"priorityClassName": "low-priority-worker"}}, - ], - ) - def test_priority_class_name(self, workers_values): + def test_priority_class_name(self): docs = render_chart( values={ "executor": "CeleryExecutor,KubernetesExecutor", @@ -445,7 +437,7 @@ def test_priority_class_name(self, workers_values): "statsd": {"priorityClassName": "low-priority-statsd"}, "triggerer": {"priorityClassName": "low-priority-triggerer"}, "dagProcessor": {"priorityClassName": "low-priority-dag-processor"}, - "workers": workers_values, + "workers": {"celery": {"priorityClassName": "low-priority-worker"}}, "cleanup": {"enabled": True, "priorityClassName": "low-priority-airflow-cleanup-pods"}, "databaseCleanup": {"enabled": True, "priorityClassName": "low-priority-database-cleanup"}, "migrateDatabaseJob": {"priorityClassName": "low-priority-run-airflow-migrations"}, diff --git a/chart/tests/helm_tests/airflow_aux/test_annotations.py b/chart/tests/helm_tests/airflow_aux/test_annotations.py index 8916a8cc3c83a..5b2273ff2e235 100644 --- a/chart/tests/helm_tests/airflow_aux/test_annotations.py +++ b/chart/tests/helm_tests/airflow_aux/test_annotations.py @@ -105,21 +105,6 @@ class TestServiceAccountAnnotations: "example": "api-server", }, ), - ( - { - "workers": { - "serviceAccount": { - "annotations": { - "example": "worker", - }, - }, - }, - }, - "templates/workers/worker-serviceaccount.yaml", - { - "example": "worker", - }, - ), ( { "workers": { @@ -137,73 +122,10 @@ class TestServiceAccountAnnotations: "example": "worker", }, ), - ( - { - "workers": { - "serviceAccount": { - "annotations": { - "worker": "example", - }, - }, - "celery": { - "serviceAccount": { - "annotations": { - "example": "worker", - }, - } - }, - }, - }, - "templates/workers/worker-serviceaccount.yaml", - { - "example": "worker", - }, - ), - ( - { - "executor": "KubernetesExecutor", - "workers": { - "serviceAccount": { - "annotations": { - "example": "worker", - }, - }, - "kubernetes": {"serviceAccount": {"create": True}}, - }, - }, - "templates/workers/worker-kubernetes-serviceaccount.yaml", - { - "example": "worker", - }, - ), - ( - { - "executor": "KubernetesExecutor", - "workers": { - "kubernetes": { - "serviceAccount": { - "create": True, - "annotations": { - "example": "worker", - }, - } - }, - }, - }, - "templates/workers/worker-kubernetes-serviceaccount.yaml", - { - "example": "worker", - }, - ), ( { "executor": "KubernetesExecutor", "workers": { - "serviceAccount": { - "annotations": { - "worker": "example", - }, - }, "kubernetes": { "serviceAccount": { "create": True, @@ -438,25 +360,6 @@ def test_tpl_rendered_annotations_celery_worker(self): assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" def test_tpl_rendered_annotations_kubernetes_worker(self): - """Test KubernetesExecutor worker SA annotations support tpl rendering.""" - k8s_objects = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": { - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-worker@project.iam", - }, - }, - }, - }, - show_only=["templates/workers/worker-serviceaccount.yaml"], - ) - assert len(k8s_objects) == 1 - annotations = k8s_objects[0]["metadata"]["annotations"] - assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" - - def test_tpl_rendered_annotations_kubernetes_worker_separate(self): """Test worker-kubernetes-serviceaccount.yaml support tpl rendering.""" k8s_objects = render_chart( values={ @@ -508,19 +411,6 @@ def test_tpl_rendered_annotations_kubernetes_worker_separate(self): "example": "release-name-api-server", }, ), - ( - { - "workers": { - "podAnnotations": { - "example": "{{ .Release.Name }}-worker", - }, - }, - }, - "templates/workers/worker-deployment.yaml", - { - "example": "release-name-worker", - }, - ), ( { "workers": { @@ -536,24 +426,6 @@ def test_tpl_rendered_annotations_kubernetes_worker_separate(self): "example": "release-name-worker", }, ), - ( - { - "workers": { - "podAnnotations": { - "test": "test", - }, - "celery": { - "podAnnotations": { - "example": "{{ .Release.Name }}-worker", - }, - }, - }, - }, - "templates/workers/worker-deployment.yaml", - { - "example": "release-name-worker", - }, - ), ( { "flower": { diff --git a/chart/tests/helm_tests/airflow_aux/test_container_lifecycle.py b/chart/tests/helm_tests/airflow_aux/test_container_lifecycle.py index 21aaa5f54a278..a1c2c63f6137a 100644 --- a/chart/tests/helm_tests/airflow_aux/test_container_lifecycle.py +++ b/chart/tests/helm_tests/airflow_aux/test_container_lifecycle.py @@ -105,37 +105,15 @@ def test_global_setting_external(self, hook_type): assert jmespath.search("spec.template.spec.containers[0].lifecycle", doc) != LIFECYCLE_PARSED # .containerLifecycleWebhooks > containerLifecycleWebhooks - @pytest.mark.parametrize( - ("hook_type", "workers_values"), - [ - ("preStop", {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}), - ("preStop", {"celery": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}}), - ( - "preStop", - { - "containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}, - "celery": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}, - }, - ), - ("postStart", {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}), - ("postStart", {"celery": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}}), - ( - "postStart", - { - "containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}, - "celery": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}, - }, - ), - ], - ) - def test_check_main_container_setting(self, hook_type, workers_values): + @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) + def test_check_main_container_setting(self, hook_type): docs = render_chart( name=RELEASE_NAME, values={ "containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}, "flower": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}, "scheduler": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}, - "workers": workers_values, + "workers": {"celery": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}}, "migrateDatabaseJob": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}, "triggerer": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}, "redis": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}}, @@ -188,74 +166,25 @@ def test_metrics_explorer_container_setting(self, hook_type): ) # Test container lifecycle hooks for worker-kerberos main container - @pytest.mark.parametrize( - ("workers_values", "expected_hook_type"), - [ - ( - { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}, - } - }, - "preStop", - ), - ( - { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}, - } - }, - "postStart", - ), - ( - { - "celery": { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}, - } - } - }, - "preStop", - ), - ( - { + @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) + def test_worker_kerberos_container_setting(self, hook_type): + docs = render_chart( + name=RELEASE_NAME, + values={ + "workers": { "celery": { "kerberosSidecar": { "enabled": True, - "containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}, + "containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}, } } - }, - "postStart", - ), - ( - { - "kerberosSidecar": { - "containerLifecycleHooks": {"postStart": {"exec": {"command": ["test"]}}} - }, - "celery": { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}, - } - }, - }, - "preStop", - ), - ], - ) - def test_worker_kerberos_container_setting(self, workers_values, expected_hook_type): - docs = render_chart( - name=RELEASE_NAME, - values={"workers": workers_values}, + } + }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.spec.containers[2].lifecycle", docs[0]) == { - expected_hook_type: LIFECYCLE_PARSED + hook_type: LIFECYCLE_PARSED } # Test container lifecycle hooks for log-groomer-sidecar main container @@ -278,60 +207,22 @@ def test_log_groomer_sidecar_container_setting_scheduler(self, hook_type): == LIFECYCLE_PARSED ) - @pytest.mark.parametrize( - ("workers_values", "expected"), - [ - ( - {"logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}}, - {"preStop": LIFECYCLE_PARSED}, - ), - ( - {"logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}}, - {"postStart": LIFECYCLE_PARSED}, - ), - ( - { - "celery": { - "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} - } - }, - {"preStop": LIFECYCLE_PARSED}, - ), - ( - { - "celery": { - "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} - } - }, - {"postStart": LIFECYCLE_PARSED}, - ), - ( - { - "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}, - "celery": { - "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} - }, - }, - {"preStop": LIFECYCLE_PARSED}, - ), - ( - { - "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}, - "celery": { - "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} - }, - }, - {"postStart": LIFECYCLE_PARSED}, - ), - ], - ) - def test_log_groomer_sidecar_container_setting(self, workers_values, expected): + @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) + def test_log_groomer_sidecar_container_setting(self, hook_type): docs = render_chart( name=RELEASE_NAME, - values={"workers": workers_values}, + values={ + "workers": { + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}} + } + } + }, show_only=[ "templates/workers/worker-deployment.yaml", ], ) - assert jmespath.search("spec.template.spec.containers[1].lifecycle", docs[0]) == expected + assert jmespath.search("spec.template.spec.containers[1].lifecycle", docs[0]) == { + hook_type: LIFECYCLE_PARSED + } diff --git a/chart/tests/helm_tests/airflow_aux/test_pod_template_file.py b/chart/tests/helm_tests/airflow_aux/test_pod_template_file.py index b65ef577d4596..701c6c528c2ab 100644 --- a/chart/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/chart/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -460,78 +460,27 @@ def test_global_node_selector(self): assert jmespath.search("kind", docs[0]) == "Pod" assert jmespath.search("spec.nodeSelector", docs[0]) == {"diskType": "ssd"} - @pytest.mark.parametrize( - "workers_values", - [ - { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, + def test_workers_affinity(self): + docs = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": { + "kubernetes": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } ] } - ] - } - } - }, - }, - { - "kubernetes": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] - } - } - }, - } - }, - { - "affinity": { - "podAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "topologyKey": "foo", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - }, - "weight": 1, } - ] + }, } }, - "kubernetes": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] - } - } - }, - }, - }, - ], - ) - def test_workers_affinity(self, workers_values): - docs = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": workers_values, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -552,36 +501,22 @@ def test_workers_affinity(self, workers_values): } } - @pytest.mark.parametrize( - "workers_values", - [ - { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - }, - { - "kubernetes": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ] - }, - }, - { - "tolerations": [{"key": "pods", "operator": "Exists", "effect": "PreferNoSchedule"}], - "kubernetes": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ] - }, - }, - ], - ) - def test_workers_tolerations(self, workers_values): + def test_workers_tolerations(self): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": workers_values, + "workers": { + "kubernetes": { + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ] + }, + }, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -592,56 +527,23 @@ def test_workers_tolerations(self, workers_values): {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - }, - { - "kubernetes": { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - } - }, - { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "not-me", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, + def test_workers_topology_spread_constraints(self): + docs = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": { + "kubernetes": { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], } - ], - "kubernetes": { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], }, }, - ], - ) - def test_workers_topology_spread_constraints(self, workers_values): - docs = render_chart( - values={"executor": "KubernetesExecutor", "workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -656,19 +558,11 @@ def test_workers_topology_spread_constraints(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"nodeSelector": {"diskType": "ssd"}}, - {"kubernetes": {"nodeSelector": {"diskType": "ssd"}}}, - {"nodeSelector": {"ssd": "diskType"}, "kubernetes": {"nodeSelector": {"diskType": "ssd"}}}, - ], - ) - def test_workers_node_selector(self, workers_values): + def test_workers_node_selector(self): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": workers_values, + "workers": {"kubernetes": {"nodeSelector": {"diskType": "ssd"}}}, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -693,7 +587,7 @@ def test_affinity_overwrite(self): } docs = render_chart( values={ - "workers": {"affinity": expected_affinity}, + "workers": {"kubernetes": {"affinity": expected_affinity}}, "affinity": { "nodeAffinity": { "preferredDuringSchedulingIgnoredDuringExecution": [ @@ -720,9 +614,16 @@ def test_tolerations_overwrite(self): docs = render_chart( values={ "workers": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], + "kubernetes": { + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ], + } }, "tolerations": [ {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} @@ -747,7 +648,7 @@ def test_topology_spread_constraints_overwrite(self): docs = render_chart( values={ "workers": { - "topologySpreadConstraints": [expected_topology_spread_constraints], + "kubernetes": {"topologySpreadConstraints": [expected_topology_spread_constraints]}, }, "topologySpreadConstraints": [ { @@ -767,17 +668,10 @@ def test_topology_spread_constraints_overwrite(self): "spec.topologySpreadConstraints", docs[0] ) - @pytest.mark.parametrize( - "workers_values", - [ - {"nodeSelector": {"diskType": "ssd"}}, - {"kubernetes": {"nodeSelector": {"diskType": "ssd"}}}, - ], - ) - def test_node_selector_overwrite(self, workers_values): + def test_node_selector_overwrite(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"kubernetes": {"nodeSelector": {"diskType": "ssd"}}}, "nodeSelector": {"type": "not-me"}, }, show_only=["templates/pod-template-file.yaml"], @@ -790,16 +684,8 @@ def test_node_selector_overwrite(self, workers_values): @pytest.mark.parametrize( ("base_scheduler_name", "worker_values", "expected"), [ - ("default-scheduler", {"schedulerName": "most-allocated"}, "most-allocated"), ("default-scheduler", {"kubernetes": {"schedulerName": "most-allocated"}}, "most-allocated"), - ( - "default-scheduler", - {"schedulerName": "least-allocated", "kubernetes": {"schedulerName": "most-allocated"}}, - "most-allocated", - ), - ("default-scheduler", {"schedulerName": None}, "default-scheduler"), ("default-scheduler", {"kubernetes": {"schedulerName": None}}, "default-scheduler"), - (None, {"schedulerName": None}, None), (None, {"kubernetes": {"schedulerName": None}}, None), ], ) @@ -849,8 +735,6 @@ def test_container_security_context_default(self): [ {"securityContext": {"runAsUser": 10}}, {"securityContexts": {"pod": {"runAsUser": 10}}}, - {"workers": {"securityContext": {"runAsUser": 10}}}, - {"workers": {"securityContexts": {"pod": {"runAsUser": 10}}}}, {"workers": {"kubernetes": {"securityContexts": {"pod": {"runAsUser": 10}}}}}, ], ) @@ -867,7 +751,6 @@ def test_pod_security_context_set(self, values): "values", [ {"securityContexts": {"containers": {"allowPrivilegeEscalation": False}}}, - {"workers": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, { "workers": { "kubernetes": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}} @@ -889,34 +772,11 @@ def test_container_security_context_set(self, values): @pytest.mark.parametrize( "values", [ - {"securityContext": {"runAsUser": 5}, "workers": {"securityContext": {"runAsUser": 10}}}, - { - "securityContexts": {"pod": {"runAsUser": 5}}, - "workers": {"securityContexts": {"pod": {"runAsUser": 10}}}, - }, { "securityContexts": {"pod": {"runAsUser": 5}}, "workers": {"kubernetes": {"securityContexts": {"pod": {"runAsUser": 10}}}}, }, - { - "workers": { - "securityContexts": {"pod": {"runAsUser": 5}}, - "kubernetes": {"securityContexts": {"pod": {"runAsUser": 10}}}, - }, - }, {"securityContext": {"runAsUser": 5}, "securityContexts": {"pod": {"runAsUser": 10}}}, - { - "workers": { - "securityContext": {"runAsUser": 5}, - "securityContexts": {"pod": {"runAsUser": 10}}, - } - }, - { - "workers": { - "securityContext": {"runAsUser": 5}, - "kubernetes": {"securityContexts": {"pod": {"runAsUser": 10}}}, - } - }, ], ) def test_pod_security_context_overwrite(self, values): @@ -928,30 +788,14 @@ def test_pod_security_context_overwrite(self, values): assert jmespath.search("spec.securityContext", docs[0]) == {"runAsUser": 10} - @pytest.mark.parametrize( - "values", - [ - { - "securityContexts": {"containers": {"allowPrivilegeEscalation": True}}, - "workers": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}, - }, - { + def test_container_security_context_overwrite(self): + docs = render_chart( + values={ "securityContexts": {"containers": {"allowPrivilegeEscalation": True}}, "workers": { "kubernetes": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}} }, }, - { - "workers": { - "securityContexts": {"container": {"allowPrivilegeEscalation": True}}, - "kubernetes": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}, - }, - }, - ], - ) - def test_container_security_context_overwrite(self, values): - docs = render_chart( - values=values, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -978,20 +822,15 @@ def test_should_add_uid_to_the_pod_template(self): assert jmespath.search("spec.securityContext.runAsUser", docs[0]) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, - {"kubernetes": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}}, - { - "extraVolumes": [{"name": "test", "emptyDir": {}}], - "kubernetes": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, - }, - ], - ) - def test_should_create_valid_volume(self, workers_values): + def test_should_create_valid_volume(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "kubernetes": { + "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}] + } + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -999,32 +838,17 @@ def test_should_create_valid_volume(self, workers_values): # [2:] -> Skipping logs and config volumes assert jmespath.search("spec.volumes[2:].name", docs[0]) == ["test-volume-airflow"] - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], - }, - { - "kubernetes": { - "extraVolumeMounts": [ - {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], + def test_should_create_valid_volume_mount(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } } }, - { - "extraVolumeMounts": [{"name": "test", "mountPath": "test"}], - "kubernetes": { - "extraVolumeMounts": [ - {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - }, - }, - ], - ) - def test_should_create_valid_volume_mount(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1084,12 +908,8 @@ def test_airflow_pod_annotations(self): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"safeToEvict": True}, "true"), ({"kubernetes": {"safeToEvict": True}}, "true"), - ({"safeToEvict": False, "kubernetes": {"safeToEvict": True}}, "true"), - ({"safeToEvict": False}, "false"), ({"kubernetes": {"safeToEvict": False}}, "false"), - ({"safeToEvict": True, "kubernetes": {"safeToEvict": False}}, "false"), ], ) def test_safe_to_evict_annotation(self, workers_values, expected): @@ -1101,18 +921,11 @@ def test_safe_to_evict_annotation(self, workers_values, expected): annotations = jmespath.search("metadata.annotations", docs[0]) assert annotations == {"cluster-autoscaler.kubernetes.io/safe-to-evict": expected} - @pytest.mark.parametrize( - "workers_values", - [ - {"safeToEvict": False}, - {"kubernetes": {"safeToEvict": False}}, - ], - ) - def test_safe_to_evict_annotation_other_services(self, workers_values): + def test_safe_to_evict_annotation_other_services(self): """Workers' safeToEvict value should not overwrite safeToEvict value of other services.""" docs = render_chart( values={ - "workers": workers_values, + "workers": {"kubernetes": {"safeToEvict": False}}, "scheduler": {"safeToEvict": True}, "triggerer": {"safeToEvict": True}, "executor": "KubernetesExecutor", @@ -1138,40 +951,22 @@ def test_global_pod_annotations_templated(self): assert jmespath.search("metadata.annotations", docs[0])["global"] == "release-name" - @pytest.mark.parametrize( - "workers_values", - [ - {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}, - {"kubernetes": {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}}, - { - "podAnnotations": {"test": "test"}, - "kubernetes": {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}, - }, - ], - ) - def test_workers_pod_annotations_templated(self, workers_values): + def test_workers_pod_annotations_templated(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}}}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("metadata.annotations", docs[0])["my_annotation"] == "release-name" - @pytest.mark.parametrize( - "workers_values", - [ - {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}, - {"kubernetes": {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}}, - ], - ) - def test_workers_pod_annotations_override(self, workers_values): + def test_workers_pod_annotations_override(self): # Worker-specific pod annotations should override global airflowPodAnnotations, # whether they come from workers.podAnnotations or workers.kubernetes.podAnnotations docs = render_chart( values={ "airflowPodAnnotations": {"my_annotation": "airflowPodAnnotations"}, - "workers": workers_values, + "workers": {"kubernetes": {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}}, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1179,18 +974,11 @@ def test_workers_pod_annotations_override(self, workers_values): assert jmespath.search("metadata.annotations", docs[0])["my_annotation"] == "workerPodAnnotations" - @pytest.mark.parametrize( - "workers_values", - [ - {"podAnnotations": {"local": "workerPodAnnotations"}}, - {"kubernetes": {"podAnnotations": {"local": "workerPodAnnotations"}}}, - ], - ) - def test_pod_annotations_merge(self, workers_values): + def test_pod_annotations_merge(self): docs = render_chart( values={ "airflowPodAnnotations": {"global": "airflowPodAnnotations"}, - "workers": workers_values, + "workers": {"kubernetes": {"podAnnotations": {"local": "workerPodAnnotations"}}}, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1200,34 +988,17 @@ def test_pod_annotations_merge(self, workers_values): assert annotations["global"] == "airflowPodAnnotations" assert annotations["local"] == "workerPodAnnotations" - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - { - "kubernetes": { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + } } }, - { - "extraInitContainers": [{"name": "test", "image": "repo:tag"}], - "kubernetes": { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - }, - ], - ) - def test_should_add_extra_init_containers(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1239,20 +1010,15 @@ def test_should_add_extra_init_containers(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, - {"kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, - { - "extraInitContainers": [{"name": "container"}], - "kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, - }, - ], - ) - def test_should_template_extra_init_containers(self, workers_values): + def test_should_template_extra_init_containers(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "kubernetes": { + "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}] + } + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1263,30 +1029,17 @@ def test_should_template_extra_init_containers(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"extraContainers": [{"name": "test-container", "image": "test-registry/test-repo:test-tag"}]}, - { - "kubernetes": { - "extraContainers": [ - {"name": "test-container", "image": "test-registry/test-repo:test-tag"} - ] + def test_should_add_extra_containers(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "extraContainers": [ + {"name": "test-container", "image": "test-registry/test-repo:test-tag"} + ] + } } }, - { - "extraContainers": [{"name": "container", "image": "repo:tag"}], - "kubernetes": { - "extraContainers": [ - {"name": "test-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - }, - ], - ) - def test_should_add_extra_containers(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1298,20 +1051,13 @@ def test_should_add_extra_containers(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}, - {"kubernetes": {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}}, - { - "extraContainers": [{"name": "{{ .Release.Name }}-test"}], - "kubernetes": {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}, - }, - ], - ) - def test_should_template_extra_containers(self, workers_values): + def test_should_template_extra_containers(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "kubernetes": {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]} + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1337,58 +1083,25 @@ def test_should_add_pod_labels(self): "tier": "airflow", } - @pytest.mark.parametrize( - "workers_values", - [ - { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ] - }, - { - "kubernetes": { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ] + def test_should_add_extraEnvs(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ] + } } }, - { - "env": [{"name": "TEST", "value": "test"}], - "kubernetes": { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ] - }, - }, - ], - ) - def test_should_add_extraEnvs(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1406,30 +1119,15 @@ def test_should_add_extraEnvs(self, workers_values): "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, } in jmespath.search("spec.containers[0].env", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - { - "labels": {"test_label": "test_label_value"}, - }, - { - "kubernetes": { - "labels": {"test_label": "test_label_value"}, - } - }, - { - "labels": {"key": "value"}, - "kubernetes": { - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": workers_values, + "workers": { + "kubernetes": { + "labels": {"test_label": "test_label_value"}, + } + }, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1439,20 +1137,9 @@ def test_should_add_component_specific_labels(self, workers_values): assert labels["test_label"] == "test_label_value" assert "key" not in labels - @pytest.mark.parametrize( - "workers_values", - [ - {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}, - {"kubernetes": {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}}, - { - "resources": {"limits": {"memory": "1Gi", "cpu": "2"}}, - "kubernetes": {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}, - }, - ], - ) - def test_should_add_resources(self, workers_values): + def test_should_add_resources(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}}}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1471,20 +1158,13 @@ def test_empty_resources(self): ) assert jmespath.search("spec.containers[0].resources", docs[0]) == {} - @pytest.mark.parametrize( - "workers_values", - [ - {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}, - {"kubernetes": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}}, - { - "hostAliases": [{"ip": "192.168.0.1", "hostnames": ["hostname"]}], - "kubernetes": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}, - }, - ], - ) - def test_workers_host_aliases(self, workers_values): + def test_workers_host_aliases(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "kubernetes": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]} + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1493,54 +1173,27 @@ def test_workers_host_aliases(self, workers_values): {"ip": "127.0.0.2", "hostnames": ["test.hostname"]} ] - @pytest.mark.parametrize( - "workers_values", - [ - {"priorityClassName": "test-priority"}, - {"kubernetes": {"priorityClassName": "test-priority"}}, - {"priorityClassName": "test", "kubernetes": {"priorityClassName": "test-priority"}}, - ], - ) - def test_workers_priority_class_name(self, workers_values): + def test_workers_priority_class_name(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"priorityClassName": "test-priority"}}}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.priorityClassName", docs[0]) == "test-priority" - @pytest.mark.parametrize( - "workers_values", - [ - { - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "preStop", "{{ .Release.Name }}"]}} - } - }, - { - "kubernetes": { - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "preStop", "{{ .Release.Name }}"]}} + def test_workers_container_lifecycle_webhooks_are_configurable(self): + docs = render_chart( + name="test-release", + values={ + "workers": { + "kubernetes": { + "containerLifecycleHooks": { + "preStop": {"exec": {"command": ["echo", "preStop", "{{ .Release.Name }}"]}} + } } } }, - { - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "postStart", "{{ .Release.Name }}"]}} - }, - "kubernetes": { - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "preStop", "{{ .Release.Name }}"]}} - } - }, - }, - ], - ) - def test_workers_container_lifecycle_webhooks_are_configurable(self, workers_values): - docs = render_chart( - name="test-release", - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1549,53 +1202,29 @@ def test_workers_container_lifecycle_webhooks_are_configurable(self, workers_val "preStop": {"exec": {"command": ["echo", "preStop", "test-release"]}} } - @pytest.mark.parametrize( - "workers_values", - [ - {"terminationGracePeriodSeconds": 123}, - {"kubernetes": {"terminationGracePeriodSeconds": 123}}, - {"terminationGracePeriodSeconds": 1, "kubernetes": {"terminationGracePeriodSeconds": 123}}, - ], - ) - def test_termination_grace_period_seconds(self, workers_values): + def test_termination_grace_period_seconds(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"terminationGracePeriodSeconds": 123}}}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.terminationGracePeriodSeconds", docs[0]) == 123 - @pytest.mark.parametrize( - "workers_values", - [ - {"runtimeClassName": "nvidia"}, - {"kubernetes": {"runtimeClassName": "nvidia"}}, - {"runtimeClassName": "test", "kubernetes": {"runtimeClassName": "nvidia"}}, - ], - ) - def test_runtime_class_name_values_are_configurable(self, workers_values): + def test_runtime_class_name_values_are_configurable(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"runtimeClassName": "nvidia"}}}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.runtimeClassName", docs[0]) == "nvidia" - @pytest.mark.parametrize( - "workers_values", - [ - {"kerberosSidecar": {"enabled": True}}, - {"kubernetes": {"kerberosSidecar": {"enabled": True}}}, - {"kerberosSidecar": {"enabled": True}, "kubernetes": {"kerberosSidecar": {"enabled": False}}}, - ], - ) - def test_airflow_local_settings_kerberos_sidecar(self, workers_values): + def test_airflow_local_settings_kerberos_sidecar(self): docs = render_chart( values={ "airflowLocalSettings": "# Well hello!", - "workers": workers_values, + "workers": {"kubernetes": {"kerberosSidecar": {"enabled": True}}}, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1609,47 +1238,20 @@ def test_airflow_local_settings_kerberos_sidecar(self, workers_values): "readOnly": True, } in jmespath.search("spec.containers[1].volumeMounts", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosSidecar": { - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - } - }, - "kubernetes": {"kerberosSidecar": {"enabled": True}}, - }, - { - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - }, + def test_kerberos_sidecar_resources(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "kerberosSidecar": { + "enabled": True, + "resources": { + "requests": {"cpu": "1m", "memory": "2Mi"}, + }, + } } } }, - { - "kerberosSidecar": { - "resources": { - "limits": {"cpu": "30m", "memory": "40Mi"}, - } - }, - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - }, - } - }, - }, - ], - ) - def test_kerberos_sidecar_resources(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1661,118 +1263,42 @@ def test_kerberos_sidecar_resources(self, workers_values): }, } - @pytest.mark.parametrize( - ("workers_values", "expected_hook_type"), - [ - ( - { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - "preStop", - ), - ( - { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - "postStart", - ), - ( - { - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - } - }, - "preStop", - ), - ( - { + @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) + def test_kerberos_sidecar_lifecycle(self, hook_type): + docs = render_chart( + name="test-release", + values={ + "workers": { "kubernetes": { "kerberosSidecar": { "enabled": True, "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} + hook_type: {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} }, } } - }, - "postStart", - ), - ( - { - "kerberosSidecar": { - "containerLifecycleHooks": {"postStart": {"exec": {"command": ["test"]}}} - }, - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": { - "preStop": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - }, - "preStop", - ), - ], - ) - def test_kerberos_sidecar_lifecycle(self, workers_values, expected_hook_type): - docs = render_chart( - name="test-release", - values={"workers": workers_values}, + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.containers[1].lifecycle", docs[0]) == { - expected_hook_type: {"exec": {"command": ["echo", "test-release"]}} + hook_type: {"exec": {"command": ["echo", "test-release"]}} } - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - }, - { - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + def test_kerberos_sidecar_security_context(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "kerberosSidecar": { + "enabled": True, + "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + } } } }, - { - "kerberosSidecar": {"securityContexts": {"container": {"runAsUser": 10}}}, - "kubernetes": { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - }, - }, - ], - ) - def test_kerberos_sidecar_security_context(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1797,17 +1323,10 @@ def test_kerberos_init_container_default_different_versions(self): assert jmespath.search("spec.initContainers[?name=='kerberos-init']", docs[0]) is None - @pytest.mark.parametrize( - "workers_values", - [ - {"kerberosInitContainer": {"enabled": True}}, - {"kubernetes": {"kerberosInitContainer": {"enabled": True}}}, - ], - ) - def test_kerberos_init_container_enable(self, workers_values): + def test_kerberos_init_container_enable(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"kubernetes": {"kerberosInitContainer": {"enabled": True}}}, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1834,18 +1353,8 @@ def test_kerberos_init_container_name_and_args_default(self): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"command": ["test", "command", "to", "run"]}, ["test", "command", "to", "run"]), - ({"command": ["cmd", "{{ .Release.Name }}"]}, ["cmd", "release-name"]), ({"kubernetes": {"command": ["test", "command", "to", "run"]}}, ["test", "command", "to", "run"]), ({"kubernetes": {"command": ["cmd", "{{ .Release.Name }}"]}}, ["cmd", "release-name"]), - ( - {"command": ["test"], "kubernetes": {"command": ["test", "command", "to", "run"]}}, - ["test", "command", "to", "run"], - ), - ( - {"command": ["test"], "kubernetes": {"command": ["cmd", "{{ .Release.Name }}"]}}, - ["cmd", "release-name"], - ), ], ) def test_should_add_command(self, workers_values, expected): @@ -1860,8 +1369,6 @@ def test_should_add_command(self, workers_values, expected): @pytest.mark.parametrize( "workers_values", [ - {"command": None}, - {"command": []}, {"kubernetes": {"command": None}}, {"kubernetes": {"command": []}}, ], @@ -1886,10 +1393,8 @@ def test_should_not_add_command_by_default(self): @pytest.mark.parametrize( ("workers_values", "kerberos_init_container", "expected_config_name"), [ - ({"kerberosSidecar": {"enabled": True}}, False, "api-server-config"), ({"kubernetes": {"kerberosSidecar": {"enabled": True}}}, False, "api-server-config"), ({"kubernetes": {"kerberosInitContainer": {"enabled": True}}}, True, "api-server-config"), - ({"kerberosInitContainer": {"enabled": True}}, True, "api-server-config"), ], ) def test_api_server_config_for_kerberos( @@ -1917,8 +1422,7 @@ def test_api_server_config_for_kerberos( @pytest.mark.parametrize( "workers_values", [ - {"kerberosSidecar": {"enabled": True}}, - {"kerberosInitContainer": {"enabled": True}}, + {"kubernetes": {"kerberosSidecar": {"enabled": True}}}, {"kubernetes": {"kerberosInitContainer": {"enabled": True}}}, ], ) @@ -1934,52 +1438,20 @@ def test_base_contains_kerberos_env(self, workers_values): scheduler_env = jmespath.search("spec.containers[0].env[*].name", docs[0]) assert set(["KRB5_CONFIG", "KRB5CCNAME"]).issubset(scheduler_env) - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - } - }, - "kubernetes": {"kerberosInitContainer": {"enabled": True}}, - }, - { - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - } - } - }, - { - "kerberosInitContainer": { - "resources": { - "requests": {"cpu": "10m", "memory": "20Mi"}, - "limits": {"cpu": "30m", "memory": "40Mi"}, - } - }, - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - } - }, - }, - ], - ) - def test_kerberos_init_container_resources(self, workers_values): + def test_kerberos_init_container_resources(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "kubernetes": { + "kerberosInitContainer": { + "enabled": True, + "resources": { + "requests": {"cpu": "1m", "memory": "2Mi"}, + "limits": {"cpu": "3m", "memory": "4Mi"}, + }, + } + } + }, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -1996,40 +1468,18 @@ def test_kerberos_init_container_resources(self, workers_values): }, } - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, - } - }, - { - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, + def test_kerberos_init_container_security_context(self): + docs = render_chart( + values={ + "workers": { + "kubernetes": { + "kerberosInitContainer": { + "enabled": True, + "securityContexts": {"container": {"runAsUser": 2000}}, + } } } }, - { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - }, - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, - } - }, - }, - ], - ) - def test_kerberos_init_container_security_context(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -2038,48 +1488,20 @@ def test_kerberos_init_container_security_context(self, workers_values): "spec.initContainers[?name=='kerberos-init'] | [0].securityContext", docs[0] ) == {"runAsUser": 2000} - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - { - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - } - }, - { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "base"]}}}, - }, - "kubernetes": { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - }, - ], - ) - def test_kerberos_init_container_lifecycle_hooks(self, workers_values): + def test_kerberos_init_container_lifecycle_hooks(self): docs = render_chart( name="test-release", values={ - "workers": workers_values, + "workers": { + "kubernetes": { + "kerberosInitContainer": { + "enabled": True, + "containerLifecycleHooks": { + "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} + }, + } + } + }, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -2096,7 +1518,7 @@ def test_service_account_name_default(self): chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.serviceAccountName", docs[0]) == "test-release-airflow-worker" + assert jmespath.search("spec.serviceAccountName", docs[0]) == "test-release-airflow-worker-kubernetes" def test_dedicated_service_account_name_default(self): docs = render_chart( diff --git a/chart/tests/helm_tests/airflow_core/test_pdb_worker.py b/chart/tests/helm_tests/airflow_core/test_pdb_worker.py index 2e18abad536df..5f98db8ae2b3b 100644 --- a/chart/tests/helm_tests/airflow_core/test_pdb_worker.py +++ b/chart/tests/helm_tests/airflow_core/test_pdb_worker.py @@ -17,69 +17,38 @@ from __future__ import annotations import jmespath -import pytest from chart_utils.helm_template_generator import render_chart class TestWorkerPdb: """Tests Worker PDB.""" - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True}}, - {"celery": {"podDisruptionBudget": {"enabled": True}}}, - ], - ) - def test_pod_disruption_budget_enabled(self, workers_values): + def test_pod_disruption_budget_enabled(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"podDisruptionBudget": {"enabled": True}}}}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) assert len(docs) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True}}, - {"celery": {"podDisruptionBudget": {"enabled": True}}}, - ], - ) - def test_pod_disruption_budget_name(self, workers_values): + def test_pod_disruption_budget_name(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"podDisruptionBudget": {"enabled": True}}}}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) assert jmespath.search("metadata.name", docs[0]) == "release-name-worker-pdb" - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True}, "labels": {"test_label": "test_label_value"}}, - { - "celery": {"podDisruptionBudget": {"enabled": True}}, - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": { - "podDisruptionBudget": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, + def test_should_add_component_specific_labels(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "podDisruptionBudget": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } } }, - { - "labels": {"key": "value"}, - "celery": { - "podDisruptionBudget": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) @@ -87,68 +56,40 @@ def test_should_add_component_specific_labels(self, workers_values): assert labels["test_label"] == "test_label_value" assert "key" not in labels - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True}}, - {"celery": {"podDisruptionBudget": {"enabled": True}}}, - ], - ) - def test_pod_disruption_budget_config_max_unavailable_default(self, workers_values): + def test_pod_disruption_budget_config_max_unavailable_default(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"podDisruptionBudget": {"enabled": True}}}}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) assert jmespath.search("spec.maxUnavailable", docs[0]) == 1 assert jmespath.search("spec.minAvailable", docs[0]) is None - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 2}}}, - {"celery": {"podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 2}}}}, - { - "podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 3}}, - "celery": {"podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 2}}}, - }, - ], - ) - def test_pod_disruption_budget_config_max_unavailable_overwrite(self, workers_values): + def test_pod_disruption_budget_config_max_unavailable_overwrite(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": {"podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 2}}} + } + }, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) assert jmespath.search("spec.maxUnavailable", docs[0]) == 2 assert jmespath.search("spec.minAvailable", docs[0]) is None - @pytest.mark.parametrize( - "workers_values", - [ - {"podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": None, "minAvailable": 1}}}, - { - "celery": { - "podDisruptionBudget": { - "enabled": True, - "config": {"maxUnavailable": None, "minAvailable": 1}, + def test_pod_disruption_budget_config_min_available_set(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "podDisruptionBudget": { + "enabled": True, + "config": {"maxUnavailable": None, "minAvailable": 1}, + } } } }, - { - "podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 2, "minAvailable": 3}}, - "celery": { - "podDisruptionBudget": { - "enabled": True, - "config": {"maxUnavailable": None, "minAvailable": 1}, - } - }, - }, - ], - ) - def test_pod_disruption_budget_config_min_available_set(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) diff --git a/chart/tests/helm_tests/airflow_core/test_scheduler.py b/chart/tests/helm_tests/airflow_core/test_scheduler.py index 140bb5153aceb..8a6128094adaa 100644 --- a/chart/tests/helm_tests/airflow_core/test_scheduler.py +++ b/chart/tests/helm_tests/airflow_core/test_scheduler.py @@ -28,7 +28,6 @@ class TestScheduler: @pytest.mark.parametrize( ("executor", "workers_values", "kind"), [ - # Test workers.celery.persistence.enabled flag ("CeleryExecutor", {"celery": {"persistence": {"enabled": False}}}, "Deployment"), ("CeleryExecutor", {"celery": {"persistence": {"enabled": True}}}, "Deployment"), ( @@ -44,14 +43,6 @@ class TestScheduler: "StatefulSet", ), ("LocalExecutor", {"celery": {"persistence": {"enabled": False}}}, "Deployment"), - # Test workers.persistence.enabled flag when celery one is default - ("CeleryExecutor", {"persistence": {"enabled": False}}, "Deployment"), - ("CeleryExecutor", {"persistence": {"enabled": True}}, "Deployment"), - ("CeleryExecutor,KubernetesExecutor", {"persistence": {"enabled": True}}, "Deployment"), - ("KubernetesExecutor", {"persistence": {"enabled": True}}, "Deployment"), - ("LocalExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), - ("LocalExecutor,KubernetesExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), - ("LocalExecutor", {"persistence": {"enabled": False}}, "Deployment"), ], ) def test_scheduler_kind(self, executor, workers_values, kind): @@ -820,20 +811,12 @@ def test_dags_mount_and_gitsync_expected_with_dag_processor(self, executor, skip c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) ] - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"annotations": {"foo": "bar"}}}, - {"celery": {"persistence": {"annotations": {"foo": "bar"}}}}, - { - "persistence": {"annotations": {"a": "b"}}, - "celery": {"persistence": {"annotations": {"foo": "bar"}}}, - }, - ], - ) - def test_persistence_volume_annotations(self, workers_values): + def test_persistence_volume_annotations(self): docs = render_chart( - values={"executor": "LocalExecutor", "workers": workers_values}, + values={ + "executor": "LocalExecutor", + "workers": {"celery": {"persistence": {"annotations": {"foo": "bar"}}}}, + }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) assert jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0]) == {"foo": "bar"} @@ -881,32 +864,15 @@ def test_scheduler_pod_hostaliases(self): assert jmespath.search("spec.template.spec.hostAliases[0].ip", docs[0]) == "127.0.0.1" assert jmespath.search("spec.template.spec.hostAliases[0].hostnames[0]", docs[0]) == "foo.local" - @pytest.mark.parametrize( - "workers_values", - [ - { - "persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}, - "celery": {"enabled": True}, - }, - { - "celery": { - "enabled": True, - "persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}, - } - }, - { - "persistence": {"storageClassName": "{{ .Release.Name }}"}, - "celery": { - "enabled": True, - "persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}, - }, - }, - ], - ) - def test_scheduler_template_storage_class_name(self, workers_values): + def test_scheduler_template_storage_class_name(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enabled": True, + "persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}, + } + }, "logs": {"persistence": {"enabled": False}}, "executor": "LocalExecutor", }, @@ -917,33 +883,16 @@ def test_scheduler_template_storage_class_name(self, workers_values): == "release-name-storage-class" ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - "celery": {"enabled": True}, - }, - { - "celery": { - "enabled": True, - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - } - }, - { - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Retain"}}, - "celery": { - "enabled": True, - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - }, - }, - ], - ) - def test_persistent_volume_claim_retention_policy(self, workers_values): + def test_persistent_volume_claim_retention_policy(self): docs = render_chart( values={ "executor": "LocalExecutor", - "workers": workers_values, + "workers": { + "celery": { + "enabled": True, + "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, + } + }, }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -966,18 +915,10 @@ def test_scheduler_termination_grace_period_seconds(self, scheduler_values, expe ) assert expected == jmespath.search("spec.template.spec.terminationGracePeriodSeconds", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"size": "50Gi"}, "celery": {"persistence": {"size": None}}}, - {"celery": {"persistence": {"size": "50Gi"}}}, - {"persistence": {"size": "10Gi"}, "celery": {"persistence": {"size": "50Gi"}}}, - ], - ) - def test_scheduler_template_storage_size(self, workers_values): + def test_scheduler_template_storage_size(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"persistence": {"size": "50Gi"}}}, "logs": {"persistence": {"enabled": False}}, "executor": "LocalExecutor", }, diff --git a/chart/tests/helm_tests/airflow_core/test_worker.py b/chart/tests/helm_tests/airflow_core/test_worker.py index e9997e9be89ac..a432687e9cf84 100644 --- a/chart/tests/helm_tests/airflow_core/test_worker.py +++ b/chart/tests/helm_tests/airflow_core/test_worker.py @@ -28,12 +28,8 @@ class TestWorker: @pytest.mark.parametrize( ("workers_values", "kind"), [ - # Test workers.celery.persistence.enabled flag ({"celery": {"persistence": {"enabled": False}}}, "Deployment"), ({"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), - # Test workers.persistence.enabled flag when celery one is default - ({"persistence": {"enabled": False}}, "Deployment"), - ({"persistence": {"enabled": True}}, "StatefulSet"), ], ) def test_worker_kind(self, workers_values, kind): @@ -71,52 +67,20 @@ def test_revision_history_limit_global(self): assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == 8 - @pytest.mark.parametrize( - ("values", "expected"), - [ - ({"revisionHistoryLimit": 8}, 8), - ({"celery": {"revisionHistoryLimit": 8}}, 8), - ], - ) - def test_revision_history_limit(self, values, expected): + def test_revision_history_limit(self): docs = render_chart( - values={"workers": values}, + values={"workers": {"celery": {"revisionHistoryLimit": 8}}}, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected - - @pytest.mark.parametrize( - ("worker_values", "global_limit", "expected"), - [ - ({"revisionHistoryLimit": 8}, 10, 8), - ({"celery": {"revisionHistoryLimit": 8}}, 10, 8), - ({"revisionHistoryLimit": 8, "celery": {"revisionHistoryLimit": 6}}, 10, 6), - ({"revisionHistoryLimit": None, "celery": {"revisionHistoryLimit": 6}}, 10, 6), - ({"revisionHistoryLimit": 8, "celery": {"revisionHistoryLimit": None}}, 10, 8), - ({"revisionHistoryLimit": None, "celery": {"revisionHistoryLimit": None}}, 10, 10), - ], - ) - def test_revision_history_limit_overwrite(self, worker_values, global_limit, expected): - docs = render_chart( - values={ - "revisionHistoryLimit": global_limit, - "workers": worker_values, - }, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected + assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == 8 @pytest.mark.parametrize( ("worker_values", "global_limit", "expected"), [ ({}, 0, 0), - ({"revisionHistoryLimit": 0}, None, 0), ({"celery": {"revisionHistoryLimit": 0}}, None, 0), - ({"revisionHistoryLimit": 0}, 10, 0), ({"celery": {"revisionHistoryLimit": 0}}, 10, 0), - ({"revisionHistoryLimit": 0, "celery": {"revisionHistoryLimit": 0}}, 10, 0), ], ) def test_revision_history_limit_zero(self, worker_values, global_limit, expected): @@ -131,42 +95,20 @@ def test_revision_history_limit_zero(self, worker_values, global_limit, expected assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraContainers": [ - {"name": "{{ .Chart.Name }}-test-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - { - "celery": { - "extraContainers": [ - { - "name": "{{ .Chart.Name }}-test-container", - "image": "test-registry/test-repo:test-tag", - } - ] - } - }, - { - "extraContainers": [{"name": "test", "image": "repo:test"}], - "celery": { - "extraContainers": [ - { - "name": "{{ .Chart.Name }}-test-container", - "image": "test-registry/test-repo:test-tag", - } - ] - }, - }, - ], - ) - def test_should_add_extra_containers_with_template(self, workers_values): + def test_should_add_extra_containers_with_template(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "extraContainers": [ + { + "name": "{{ .Chart.Name }}-test-container", + "image": "test-registry/test-repo:test-tag", + } + ] + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -179,33 +121,15 @@ def test_should_add_extra_containers_with_template(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - "celery": {"enabled": True}, - }, - { - "celery": { - "enabled": True, - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - } - }, - { - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Retain"}}, - "celery": { - "enabled": True, - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, - }, - }, - ], - ) - def test_persistent_volume_claim_retention_policy(self, workers_values): + def test_persistent_volume_claim_retention_policy(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"}}, + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -214,16 +138,9 @@ def test_persistent_volume_claim_retention_policy(self, workers_values): "whenDeleted": "Delete", } - @pytest.mark.parametrize( - "workers_values", - [ - {"waitForMigrations": {"enabled": False}}, - {"celery": {"waitForMigrations": {"enabled": False}}}, - ], - ) - def test_disable_wait_for_migration(self, workers_values): + def test_disable_wait_for_migration(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"waitForMigrations": {"enabled": False}}}}, show_only=["templates/workers/worker-deployment.yaml"], ) actual = jmespath.search( @@ -275,34 +192,17 @@ def test_logs_mount_on_wait_for_migrations_initcontainer(self, logs_values, expe for m in mounts ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - { - "celery": { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + } } }, - { - "extraInitContainers": [{"name": "container", "image": "repo:tag"}], - "celery": { - "extraInitContainers": [ - {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ] - }, - }, - ], - ) - def test_should_add_extra_init_containers(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -314,20 +214,13 @@ def test_should_add_extra_init_containers(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, - {"celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, - { - "extraInitContainers": [{"name": "container"}], - "celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, - }, - ], - ) - def test_should_template_extra_init_containers(self, workers_values): + def test_should_template_extra_init_containers(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]} + } + }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -336,22 +229,13 @@ def test_should_template_extra_init_containers(self, workers_values): {"name": "release-name-test-init-container"} ] - @pytest.mark.parametrize( - "workers_values", - [ - {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, - {"celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}}, - { - "extraVolumes": [{"name": "test", "emptyDir": {}}], - "celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, - }, - ], - ) - def test_should_add_extra_volume(self, workers_values): + def test_should_add_extra_volume(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]} + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -359,34 +243,17 @@ def test_should_add_extra_volume(self, workers_values): # [:-1] -> Skipping config volume assert jmespath.search("spec.template.spec.volumes[:-1].name", docs[0]) == ["test-volume-airflow"] - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], - }, - { - "celery": { - "extraVolumeMounts": [ - {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - } - }, - { - "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], - "celery": { - "extraVolumeMounts": [ - {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - }, - }, - ], - ) - def test_should_add_extra_volume_mount(self, workers_values): + def test_should_add_extra_volume_mount(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -414,58 +281,25 @@ def test_should_add_global_volume_and_global_volume_mount(self): jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name", docs[0]) == "test-volume" ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ], - }, - { - "celery": { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ], + def test_should_add_extraEnvs(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], + } } }, - { - "env": [{"name": "TEST", "value": "test"}], - "celery": { - "env": [ - {"name": "TEST_ENV_1", "value": "test_env_1"}, - { - "name": "TEST_ENV_2", - "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, - }, - { - "name": "TEST_ENV_3", - "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, - }, - ], - }, - }, - ], - ) - def test_should_add_extraEnvs(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -484,20 +318,13 @@ def test_should_add_extraEnvs(self, workers_values): "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}, - {"celery": {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}}, - { - "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, - "celery": {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}, - }, - ], - ) - def test_should_add_extraEnvs_to_wait_for_migration_container(self, workers_values): + def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}} + } + }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -505,30 +332,15 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self, workers_valu "spec.template.spec.initContainers[0].env", docs[0] ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": { - "labels": {"test_label": "test_label_value"}, - } - }, - { - "labels": {"key": "value"}, - "celery": { - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "labels": {"test_label": "test_label_value"}, + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -537,20 +349,12 @@ def test_should_add_component_specific_labels(self, workers_values): assert labels["test_label"] == "test_label_value" assert "key" not in labels - @pytest.mark.parametrize( - "workers_values", - [ - {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}, - {"celery": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}}, - { - "hostAliases": [{"ip": "192.168.0.1", "hostnames": ["hostname"]}], - "celery": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}, - }, - ], - ) - def test_workers_host_aliases(self, workers_values): + def test_workers_host_aliases(self): docs = render_chart( - values={"executor": "CeleryExecutor", "workers": workers_values}, + values={ + "executor": "CeleryExecutor", + "workers": {"celery": {"hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]}}, + }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -562,15 +366,6 @@ def test_workers_host_aliases(self, workers_values): ("workers_values", "expected_update_strategy"), [ ({"celery": {"persistence": {"enabled": False}}}, None), - ({"updateStrategy": None, "celery": {"persistence": {"enabled": False}}}, None), - ( - { - "updateStrategy": {"rollingUpdate": {"partition": 0}}, - "celery": {"persistence": {"enabled": True}}, - }, - {"rollingUpdate": {"partition": 0}}, - ), - ({"updateStrategy": None, "celery": {"persistence": {"enabled": True}}}, None), ({"celery": {"updateStrategy": None, "persistence": {"enabled": False}}}, None), ( { @@ -582,16 +377,6 @@ def test_workers_host_aliases(self, workers_values): {"rollingUpdate": {"partition": 0}}, ), ({"celery": {"updateStrategy": None, "persistence": {"enabled": True}}}, None), - ( - { - "updateStrategy": {"rollingUpdate": {"partition": 1}}, - "celery": { - "updateStrategy": {"rollingUpdate": {"partition": 0}}, - "persistence": {"enabled": True}, - }, - }, - {"rollingUpdate": {"partition": 0}}, - ), ], ) def test_workers_update_strategy(self, workers_values, expected_update_strategy): @@ -610,111 +395,17 @@ def test_workers_update_strategy(self, workers_values, expected_update_strategy) [ ({"celery": {"persistence": {"enabled": True}}}, None), ( - {"celery": {"persistence": {"enabled": False}}}, - {"rollingUpdate": {"maxSurge": "100%", "maxUnavailable": "50%"}}, - ), - ({"strategy": None, "celery": {"persistence": {"enabled": False}}}, None), - ( - { - "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - "celery": {"persistence": {"enabled": False}}, - }, - {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - ), - ( - {"celery": {"strategy": None, "persistence": {"enabled": False}}}, - {"rollingUpdate": {"maxSurge": "100%", "maxUnavailable": "50%"}}, - ), - ( - { - "celery": { - "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - "persistence": {"enabled": False}, - } - }, - {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - ), - ], - ) - def test_workers_strategy(self, workers_values, expected_strategy): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": workers_values, - }, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert expected_strategy == jmespath.search("spec.strategy", docs[0]) - - @pytest.mark.parametrize( - "workers_values", - [ - { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] - } - } - }, - }, - { - "celery": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] - } - } - }, - } - }, - { - "affinity": { - "podAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "topologyKey": "foo", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - }, - "weight": 1, - } - ] + { + "celery": { + "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, + "persistence": {"enabled": False}, } }, - "celery": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] - } - } - }, - }, - }, + {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, + ), ], ) - def test_affinity(self, workers_values): + def test_workers_strategy(self, workers_values, expected_strategy): docs = render_chart( values={ "executor": "CeleryExecutor", @@ -723,6 +414,33 @@ def test_affinity(self, workers_values): show_only=["templates/workers/worker-deployment.yaml"], ) + assert expected_strategy == jmespath.search("spec.strategy", docs[0]) + + def test_affinity(self): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": { + "celery": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + } + }, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + assert jmespath.search("kind", docs[0]) == "StatefulSet" assert jmespath.search("spec.template.spec.affinity", docs[0]) == { "nodeAffinity": { @@ -738,36 +456,22 @@ def test_affinity(self, workers_values): } } - @pytest.mark.parametrize( - "workers_values", - [ - { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - }, - { - "celery": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ] - }, - }, - { - "tolerations": [{"key": "pods", "operator": "Exists", "effect": "PreferNoSchedule"}], - "celery": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ] - }, - }, - ], - ) - def test_tolerations(self, workers_values): + def test_tolerations(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ] + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -777,56 +481,22 @@ def test_tolerations(self, workers_values): {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, + def test_topology_spread_constraints(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ] } - ] - }, - { - "celery": { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ] } }, - { - "topologySpreadConstraints": [ - { - "maxSkew": 2, - "topologyKey": "not-me", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"airflow": "test"}}, - } - ], - "celery": { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ] - }, - }, - ], - ) - def test_topology_spread_constraints(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -839,19 +509,11 @@ def test_topology_spread_constraints(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - {"nodeSelector": {"diskType": "ssd"}}, - {"celery": {"nodeSelector": {"diskType": "ssd"}}}, - {"nodeSelector": {"ssd": "diskType"}, "celery": {"nodeSelector": {"diskType": "ssd"}}}, - ], - ) - def test_node_selector(self, workers_values): + def test_node_selector(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": {"celery": {"nodeSelector": {"diskType": "ssd"}}}, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -875,7 +537,7 @@ def test_affinity_overwrite(self): } docs = render_chart( values={ - "workers": {"affinity": expected_affinity}, + "workers": {"celery": {"affinity": expected_affinity}}, "affinity": { "nodeAffinity": { "preferredDuringSchedulingIgnoredDuringExecution": [ @@ -900,9 +562,16 @@ def test_tolerations_overwrite(self): docs = render_chart( values={ "workers": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], + "celery": { + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ] + }, }, "tolerations": [ {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} @@ -924,7 +593,7 @@ def test_topology_spread_constraints_overwrite(self): } docs = render_chart( values={ - "workers": {"topologySpreadConstraints": [expected_topology_spread_constraints]}, + "workers": {"celery": {"topologySpreadConstraints": [expected_topology_spread_constraints]}}, "topologySpreadConstraints": [ { "maxSkew": 1, @@ -941,17 +610,10 @@ def test_topology_spread_constraints_overwrite(self): "spec.template.spec.topologySpreadConstraints", docs[0] ) - @pytest.mark.parametrize( - "workers_values", - [ - {"nodeSelector": {"diskType": "ssd"}}, - {"celery": {"nodeSelector": {"diskType": "ssd"}}}, - ], - ) - def test_node_selector_overwrite(self, workers_values): + def test_node_selector_overwrite(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"nodeSelector": {"diskType": "ssd"}}}, "nodeSelector": {"type": "not-me"}, }, show_only=["templates/workers/worker-deployment.yaml"], @@ -962,16 +624,8 @@ def test_node_selector_overwrite(self, workers_values): @pytest.mark.parametrize( ("base_scheduler_name", "worker_values", "expected"), [ - ("default-scheduler", {"schedulerName": "most-allocated"}, "most-allocated"), ("default-scheduler", {"celery": {"schedulerName": "most-allocated"}}, "most-allocated"), - ( - "default-scheduler", - {"schedulerName": "least-allocated", "celery": {"schedulerName": "most-allocated"}}, - "most-allocated", - ), - ("default-scheduler", {"schedulerName": None}, "default-scheduler"), ("default-scheduler", {"celery": {"schedulerName": None}}, "default-scheduler"), - (None, {"schedulerName": None}, None), (None, {"celery": {"schedulerName": None}}, None), ], ) @@ -1001,17 +655,9 @@ def test_should_create_default_affinity(self): docs[0], ) == {"component": "worker"} - @pytest.mark.parametrize( - "workers_values", - [ - {"runtimeClassName": "nvidia"}, - {"celery": {"runtimeClassName": "nvidia"}}, - {"runtimeClassName": "test", "celery": {"runtimeClassName": "nvidia"}}, - ], - ) - def test_runtime_class_name_values_are_configurable(self, workers_values): + def test_runtime_class_name_values_are_configurable(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"runtimeClassName": "nvidia"}}}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1027,67 +673,18 @@ def test_livenessprobe_default_command(self): ) assert "airflow.providers.celery.executors.celery_executor.app" in livenessprobe_cmd[-1] - @pytest.mark.parametrize( - "workers_values", - [ - { - "celery": { - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "wow such test"], - } - } - }, - { - "livenessProbe": { - "initialDelaySeconds": 11, - "timeoutSeconds": 22, - "failureThreshold": 33, - "periodSeconds": 44, - "command": ["test"], - }, - "celery": { - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "wow such test"], - } - }, - }, - ], - ) - def test_livenessprobe_celery_values_overwrite(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - livenessprobe = jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0]) - assert livenessprobe == { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "exec": { - "command": ["sh", "-c", "echo", "wow such test"], - }, - } - def test_livenessprobe_values_overwrite(self): docs = render_chart( values={ "workers": { - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "wow such test"], + "celery": { + "livenessProbe": { + "initialDelaySeconds": 111, + "timeoutSeconds": 222, + "failureThreshold": 333, + "periodSeconds": 444, + "command": ["sh", "-c", "echo", "wow such test"], + } } } }, @@ -1105,16 +702,9 @@ def test_livenessprobe_values_overwrite(self): }, } - @pytest.mark.parametrize( - "workers_values", - [ - {"livenessProbe": {"enabled": False}, "celery": {"livenessProbe": {"enabled": None}}}, - {"celery": {"livenessProbe": {"enabled": False}}}, - ], - ) - def test_disable_livenessprobe(self, workers_values): + def test_disable_livenessprobe(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"livenessProbe": {"enabled": False}}}}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1174,20 +764,9 @@ def test_logs_persistence_changes_volume(self, log_values, expected_volume): assert {"name": "logs", **expected_volume} in jmespath.search("spec.template.spec.volumes", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}, - {"celery": {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}}, - { - "resources": {"limits": {"cpu": "100m", "memory": "128Mi"}}, - "celery": {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}, - }, - ], - ) - def test_worker_resources_are_configurable(self, workers_values): + def test_worker_resources_are_configurable(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}}}, show_only=["templates/workers/worker-deployment.yaml"], ) # main container @@ -1229,19 +808,11 @@ def test_airflow_local_settings(self): assert volume_mount in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) assert volume_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"kerberosSidecar": {"enabled": True}}, - {"celery": {"kerberosSidecar": {"enabled": True}}}, - {"kerberosSidecar": {"enabled": True}, "celery": {"kerberosSidecar": {"enabled": False}}}, - ], - ) - def test_airflow_local_settings_kerberos_sidecar(self, workers_values): + def test_airflow_local_settings_kerberos_sidecar(self): docs = render_chart( values={ "airflowLocalSettings": "# Well hello!", - "workers": workers_values, + "workers": {"celery": {"kerberosSidecar": {"enabled": True}}}, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1269,17 +840,10 @@ def test_kerberos_init_container_default_different_versions(self): is None ) - @pytest.mark.parametrize( - "workers_values", - [ - {"kerberosInitContainer": {"enabled": True}}, - {"celery": {"kerberosInitContainer": {"enabled": True}}}, - ], - ) - def test_kerberos_init_container_enable(self, workers_values): + def test_kerberos_init_container_enable(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"kerberosInitContainer": {"enabled": True}}}, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1306,52 +870,20 @@ def test_kerberos_init_container_name_and_args_default(self): assert initContainers["name"] == "kerberos-init" assert initContainers["args"] == ["kerberos", "-o"] - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - } - }, - "celery": {"kerberosInitContainer": {"enabled": True}}, - }, - { - "celery": { - "kerberosInitContainer": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - } - } - }, - { - "kerberosInitContainer": { - "resources": { - "requests": {"cpu": "10m", "memory": "20Mi"}, - "limits": {"cpu": "30m", "memory": "40Mi"}, - } - }, - "celery": { - "kerberosInitContainer": { - "enabled": True, - "resources": { - "requests": {"cpu": "1m", "memory": "2Mi"}, - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - } - }, - }, - ], - ) - def test_kerberos_init_container_resources(self, workers_values): + def test_kerberos_init_container_resources(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "kerberosInitContainer": { + "enabled": True, + "resources": { + "requests": {"cpu": "1m", "memory": "2Mi"}, + "limits": {"cpu": "3m", "memory": "4Mi"}, + }, + } + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1369,47 +901,21 @@ def test_kerberos_init_container_resources(self, workers_values): }, } - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - { - "celery": { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, + def test_kerberos_init_container_lifecycle_hooks(self): + docs = render_chart( + name="test-release", + values={ + "workers": { + "celery": { + "kerberosInitContainer": { + "enabled": True, + "containerLifecycleHooks": { + "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} + }, + } } } }, - { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "base"]}}}, - }, - "celery": { - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - }, - }, - ], - ) - def test_kerberos_init_container_lifecycle_hooks(self, workers_values): - docs = render_chart( - name="test-release", - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1432,11 +938,8 @@ def test_default_command_and_args_airflow_version(self): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"command": ["custom", "command"]}, ["custom", "command"]), - ({"command": ["custom", "{{ .Release.Name }}"]}, ["custom", "release-name"]), ({"celery": {"command": ["custom", "command"]}}, ["custom", "command"]), ({"celery": {"command": ["custom", "{{ .Release.Name }}"]}}, ["custom", "release-name"]), - ({"command": ["test"], "celery": {"command": ["custom", "command"]}}, ["custom", "command"]), ], ) def test_should_add_command(self, workers_values, expected): @@ -1450,8 +953,6 @@ def test_should_add_command(self, workers_values, expected): @pytest.mark.parametrize( "workers_values", [ - {"command": None}, - {"command": []}, {"celery": {"command": None}}, {"celery": {"command": []}}, ], @@ -1467,13 +968,9 @@ def test_should_not_add_command(self, workers_values): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"args": None}, None), - ({"args": []}, None), - ({"celery": {"args": None}}, ["bash", "-c", "exec \\\nairflow celery worker"]), - ({"celery": {"args": []}}, ["bash", "-c", "exec \\\nairflow celery worker"]), - ({"args": ["custom", "args", "{{ .Release.Service }}"]}, ["custom", "args", "Helm"]), + ({"celery": {"args": None}}, None), + ({"celery": {"args": []}}, None), ({"celery": {"args": ["custom", "args", "{{ .Release.Service }}"]}}, ["custom", "args", "Helm"]), - ({"args": ["test"], "celery": {"args": ["custom", "args"]}}, ["custom", "args"]), ], ) def test_should_add_args(self, workers_values, expected): @@ -1504,51 +1001,27 @@ def test_dags_gitsync_with_persistence_no_sidecar_or_init_container(self): # No gitsync sidecar or init container assert "git-sync" not in [ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0]) - ] - assert "git-sync-init" not in [ - c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) - ] - - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"annotations": {"foo": "bar"}}}, - {"celery": {"persistence": {"annotations": {"foo": "bar"}}}}, - { - "persistence": {"annotations": {"a": "b"}}, - "celery": {"persistence": {"annotations": {"foo": "bar"}}}, - }, - ], - ) - def test_persistence_volume_annotations(self, workers_values): + ] + assert "git-sync-init" not in [ + c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) + ] + + def test_persistence_volume_annotations(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"persistence": {"annotations": {"foo": "bar"}}}}}, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0]) == {"foo": "bar"} - @pytest.mark.parametrize( - "workers_values", - [ - { - "annotations": {"test_annotation": "test_annotation_value"}, - }, - { - "celery": { - "annotations": {"test_annotation": "test_annotation_value"}, + def test_should_add_component_specific_annotations(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "annotations": {"test_annotation": "test_annotation_value"}, + } } }, - { - "annotations": {"test": "test"}, - "celery": { - "annotations": {"test_annotation": "test_annotation_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_annotations(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -1560,20 +1033,8 @@ def test_should_add_component_specific_annotations(self, workers_values): ("globalScope", "localScope", "precedence"), [ ({}, {}, "false"), - ({}, {"safeToEvict": True}, "true"), ({}, {"celery": {"safeToEvict": True}}, "true"), - ({}, {"safeToEvict": False}, "false"), ({}, {"celery": {"safeToEvict": False}}, "false"), - ({}, {"safeToEvict": False, "celery": {"safeToEvict": True}}, "true"), - ({}, {"safeToEvict": True, "celery": {"safeToEvict": False}}, "false"), - ( - {}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": True, - }, - "true", - ), ( {}, { @@ -1584,14 +1045,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": False, - }, - "true", - ), ( {}, { @@ -1602,14 +1055,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": True, - }, - "false", - ), ( {}, { @@ -1620,14 +1065,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "false", ), - ( - {}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": False, - }, - "false", - ), ( {}, { @@ -1638,54 +1075,26 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - {"safeToEvict": True}, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, {"celery": {"safeToEvict": True}}, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - {"safeToEvict": False}, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, {"celery": {"safeToEvict": False}}, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - {"safeToEvict": True}, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, {"celery": {"safeToEvict": True}}, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - {"safeToEvict": False}, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, {"celery": {"safeToEvict": False}}, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": False, - }, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { @@ -1696,14 +1105,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": False, - }, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { @@ -1714,14 +1115,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": False, - }, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { @@ -1732,14 +1125,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": False, - }, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { @@ -1750,14 +1135,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": True, - }, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { @@ -1768,14 +1145,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": True, - }, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { @@ -1786,14 +1155,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "false", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, - "safeToEvict": True, - }, - "true", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { @@ -1804,14 +1165,6 @@ def test_should_add_component_specific_annotations(self, workers_values): }, "true", ), - ( - {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - { - "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, - "safeToEvict": True, - }, - "false", - ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { @@ -1836,91 +1189,33 @@ def test_safetoevict_annotations(self, globalScope, localScope, precedence): == precedence ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - }, - }, - { - "metadata": {"name": "test-volume-airflow-2"}, - "spec": { - "storageClassName": "storage-class-2", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "20Gi"}}, - }, - }, - ] - }, - { - "celery": { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, + def test_should_add_extra_volume_claim_templates(self): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": { + "celery": { + "volumeClaimTemplates": [ + { + "metadata": {"name": "test-volume-airflow-1"}, + "spec": { + "storageClassName": "storage-class-1", + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + }, }, - }, - { - "metadata": {"name": "test-volume-airflow-2"}, - "spec": { - "storageClassName": "storage-class-2", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "20Gi"}}, + { + "metadata": {"name": "test-volume-airflow-2"}, + "spec": { + "storageClassName": "storage-class-2", + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "20Gi"}}, + }, }, - }, - ] - } - }, - { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test"}, - "spec": { - "storageClassName": "storage", - "accessModes": ["ReadOnce"], - "resources": {"requests": {"storage": "1Gi"}}, - }, + ] } - ], - "celery": { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - }, - }, - { - "metadata": {"name": "test-volume-airflow-2"}, - "spec": { - "storageClassName": "storage-class-2", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "20Gi"}}, - }, - }, - ] }, }, - ], - ) - def test_should_add_extra_volume_claim_templates(self, workers_values): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": workers_values, - }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2127,10 +1422,10 @@ def test_volume_claim_templates_conditional_logic( @pytest.mark.parametrize( ("globalScope", "localScope", "precedence"), [ - ({"scope": "global"}, {"podAnnotations": {}}, "global"), - ({}, {"podAnnotations": {"scope": "local"}}, "local"), - ({"scope": "global"}, {"podAnnotations": {"scope": "local"}}, "local"), - ({}, {}, None), + ({"scope": "global"}, {"celery": {"podAnnotations": {}}}, "global"), + ({}, {"celery": {"podAnnotations": {"scope": "local"}}}, "local"), + ({"scope": "global"}, {"celery": {"podAnnotations": {"scope": "local"}}}, "local"), + ({}, {"celery": {}}, None), ], ) def test_podannotations_precedence(self, globalScope, localScope, precedence): @@ -2143,20 +1438,13 @@ def test_podannotations_precedence(self, globalScope, localScope, precedence): else: assert jmespath.search("spec.template.metadata.annotations.scope", docs[0]) is None - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}, - {"celery": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}}, - { - "persistence": {"storageClassName": "{{ .Release.Name }}"}, - "celery": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}, - }, - ], - ) - def test_worker_template_storage_class_name(self, workers_values): + def test_worker_template_storage_class_name(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}} + } + }, show_only=["templates/workers/worker-deployment.yaml"], ) assert ( @@ -2167,11 +1455,8 @@ def test_worker_template_storage_class_name(self, workers_values): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"replicas": 2}, 2), ({"celery": {"replicas": 2}}, 2), - ({"celery": {"replicas": None}}, 1), - ({"replicas": 2, "celery": {"replicas": 3}}, 3), - ({"replicas": 2, "celery": {"replicas": None}}, 2), + ({"celery": {"replicas": None}}, None), ], ) def test_workers_replicas(self, workers_values, expected): @@ -2182,18 +1467,10 @@ def test_workers_replicas(self, workers_values, expected): assert expected == jmespath.search("spec.replicas", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"size": "50Gi"}, "celery": {"persistence": {"size": None}}}, - {"celery": {"persistence": {"size": "50Gi"}}}, - {"persistence": {"size": "10Gi"}, "celery": {"persistence": {"size": "50Gi"}}}, - ], - ) - def test_template_storage_size(self, workers_values): + def test_template_storage_size(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"persistence": {"size": "50Gi"}}}, "logs": {"persistence": {"enabled": False}}, }, show_only=["templates/workers/worker-deployment.yaml"], @@ -2203,18 +1480,10 @@ def test_template_storage_size(self, workers_values): jmespath.search("spec.volumeClaimTemplates[0].spec.resources.requests.storage", docs[0]) == "50Gi" ) - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"fixPermissions": True}, "celery": {"persistence": {"fixPermissions": None}}}, - {"celery": {"persistence": {"fixPermissions": True}}}, - {"persistence": {"fixPermissions": False}, "celery": {"persistence": {"fixPermissions": True}}}, - ], - ) - def test_init_container_volume_permissions_exist(self, workers_values): + def test_init_container_volume_permissions_exist(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"persistence": {"fixPermissions": True}}}, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2224,18 +1493,10 @@ def test_init_container_volume_permissions_exist(self, workers_values): == 1 ) - @pytest.mark.parametrize( - "workers_values", - [ - {"persistence": {"fixPermissions": False}, "celery": {"persistence": {"fixPermissions": None}}}, - {"celery": {"persistence": {"fixPermissions": False}}}, - {"persistence": {"fixPermissions": True}, "celery": {"persistence": {"fixPermissions": False}}}, - ], - ) - def test_init_container_volume_permissions_not_exist(self, workers_values): + def test_init_container_volume_permissions_not_exist(self): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"persistence": {"fixPermissions": False}}}, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2252,18 +1513,8 @@ def test_pod_management_policy_default(self): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"podManagementPolicy": "Parallel"}, "Parallel"), - ({"podManagementPolicy": "OrderedReady"}, "OrderedReady"), ({"celery": {"podManagementPolicy": "Parallel"}}, "Parallel"), ({"celery": {"podManagementPolicy": "OrderedReady"}}, "OrderedReady"), - ( - {"podManagementPolicy": "OrderedReady", "celery": {"podManagementPolicy": "Parallel"}}, - "Parallel", - ), - ( - {"podManagementPolicy": "Parallel", "celery": {"podManagementPolicy": "OrderedReady"}}, - "OrderedReady", - ), ], ) def test_pod_management_policy(self, workers_values, expected): @@ -2273,44 +1524,26 @@ def test_pod_management_policy(self, workers_values, expected): assert jmespath.search("spec.podManagementPolicy", docs[0]) == expected - @pytest.mark.parametrize( - "workers_values", [{"podManagementPolicy": "Test"}, {"celery": {"podManagementPolicy": "Test"}}] - ) - def test_pod_management_policy_not_valid_value(self, workers_values): + def test_pod_management_policy_not_valid_value(self): with pytest.raises(HelmFailedError): render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"] + values={"workers": {"celery": {"podManagementPolicy": "Test"}}}, + show_only=["templates/workers/worker-deployment.yaml"], ) - @pytest.mark.parametrize( - "workers_values", - [ - {"terminationGracePeriodSeconds": 5}, - {"celery": {"terminationGracePeriodSeconds": 5}}, - {"terminationGracePeriodSeconds": 10, "celery": {"terminationGracePeriodSeconds": 5}}, - ], - ) - def test_termination_grace_period(self, workers_values): + def test_termination_grace_period(self): docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"] + values={"workers": {"celery": {"terminationGracePeriodSeconds": 5}}}, + show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.spec.terminationGracePeriodSeconds", docs[0]) == 5 - @pytest.mark.parametrize( - "workers_values", - [ - {"extraPorts": [{"name": "test-extra-port", "containerPort": 10}]}, - {"celery": {"extraPorts": [{"name": "test-extra-port", "containerPort": 10}]}}, - { - "extraPorts": [{"name": "test", "containerPort": 1}], - "celery": {"extraPorts": [{"name": "test-extra-port", "containerPort": 10}]}, - }, - ], - ) - def test_overwrite_extra_ports(self, workers_values): + def test_overwrite_extra_ports(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": {"celery": {"extraPorts": [{"name": "test-extra-port", "containerPort": 10}]}} + }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2319,13 +1552,6 @@ def test_overwrite_extra_ports(self, workers_values): ] -class TestWorkerLogGroomer(LogGroomerTestBase): - """Worker groomer.""" - - obj_name = "worker" - folder = "workers" - - class TestWorkerCeleryLogGroomer(LogGroomerTestBase): """Worker Celery groomer.""" @@ -2381,42 +1607,25 @@ def test_log_groomer_with_persistence_combinations( if expect_log_groomer: assert "worker-log-groomer" in container_names - assert len(containers) == 2 - else: - assert "worker-log-groomer" not in container_names - assert len(containers) == 1 - - -class TestWorkerKedaAutoScaler: - """Tests worker keda auto scaler.""" - - @pytest.mark.parametrize( - "workers_values", - [ - { - "celery": {"keda": {"enabled": True}}, - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": { - "keda": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - } - }, - { - "labels": {"key": "value"}, - "celery": { - "keda": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + assert len(containers) == 2 + else: + assert "worker-log-groomer" not in container_names + assert len(containers) == 1 + + +class TestWorkerKedaAutoScaler: + """Tests worker keda auto scaler.""" + + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "keda": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -2456,20 +1665,6 @@ def test_keda_query_default(self): @pytest.mark.parametrize( "workers_values", [ - # workers - custom static query - { - "keda": { - "enabled": True, - "query": "SELECT ceil(COUNT(*)::decimal / 32) FROM task_instance", - } - }, - # workers - custom template query - { - "keda": { - "enabled": True, - "query": "SELECT ceil(COUNT(*)::decimal / {{ mul .Values.config.celery.worker_concurrency 2 }}) FROM task_instance", - } - }, # workers.celery - custom static query { "celery": { @@ -2488,16 +1683,6 @@ def test_keda_query_default(self): } } }, - # workers.celery overrides workers when both are set - { - "keda": {"query": "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance"}, - "celery": { - "keda": { - "enabled": True, - "query": "SELECT ceil(COUNT(*)::decimal / 32) FROM task_instance", - } - }, - }, ], ) def test_keda_query_overwrite(self, workers_values): @@ -2531,20 +1716,11 @@ def test_mysql_db_backend_keda_worker(self): class TestWorkerHPAAutoScaler: """Tests worker HPA auto scaler.""" - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True}, "hpa": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}}, "hpa": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}, "hpa": {"enabled": True}}}, - {"keda": {"enabled": True}, "celery": {"hpa": {"enabled": True}}}, - ], - ) - def test_should_be_disabled_on_keda_enabled(self, workers_values): + def test_should_be_disabled_on_keda_enabled(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True}, "hpa": {"enabled": True}}}, }, show_only=[ "templates/workers/worker-kedaautoscaler.yaml", @@ -2554,30 +1730,16 @@ def test_should_be_disabled_on_keda_enabled(self, workers_values): assert len(docs) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - {"celery": {"hpa": {"enabled": True}}, "labels": {"test_label": "test_label_value"}}, - { - "celery": { - "hpa": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - } - }, - { - "labels": {"key": "value"}, - "celery": { - "hpa": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "hpa": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, }, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -2591,25 +1753,18 @@ def test_should_remove_replicas_field(self): values={ "executor": "CeleryExecutor", "workers": { - "hpa": {"enabled": True}, + "celery": {"hpa": {"enabled": True}}, }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert "replicas" not in jmespath.search("spec", docs[0]) - @pytest.mark.parametrize( - "workers_values", - [ - {"hpa": {"enabled": True}}, - {"celery": {"hpa": {"enabled": True}}}, - ], - ) - def test_hpa_metrics_default(self, workers_values): + def test_hpa_metrics_default(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": {"celery": {"hpa": {"enabled": True}}}, }, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -2621,74 +1776,26 @@ def test_hpa_metrics_default(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "hpa": { - "enabled": True, - "metrics": [ - { - "type": "Pods", - "pods": { - "metric": {"name": "custom"}, - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - } - ], - } - }, - { - "celery": { - "hpa": { - "enabled": True, - "metrics": [ - { - "type": "Pods", - "pods": { - "metric": {"name": "custom"}, - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - } - ], - } - } - }, - { - "hpa": { - "enabled": True, - "metrics": [ - { - "type": "Resource", - "resource": { - "name": "memory", - "target": {"type": "Utilization", "averageUtilization": 1}, - }, - } - ], - }, - "celery": { - "hpa": { - "enabled": True, - "metrics": [ - { - "type": "Pods", - "pods": { - "metric": {"name": "custom"}, - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - } - ], - } - }, - }, - ], - ) - def test_hpa_metrics_override(self, workers_values): + def test_hpa_metrics_override(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Pods", + "pods": { + "metric": {"name": "custom"}, + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ], + } + } + }, }, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -2707,27 +1814,14 @@ def test_hpa_metrics_override(self, workers_values): class TestWorkerNetworkPolicy: """Tests worker network policy.""" - @pytest.mark.parametrize( - "workers_values", - [ - { - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": {"labels": {"test_label": "test_label_value"}}, - }, - { - "labels": {"key": "value"}, - "celery": {"labels": {"test_label": "test_label_value"}}, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "networkPolicies": {"enabled": True}, "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, }, show_only=["templates/workers/worker-networkpolicy.yaml"], ) @@ -2755,26 +1849,13 @@ def test_should_allow_api_server_to_read_worker_logs(self, executor): class TestWorkerService: """Tests worker service.""" - @pytest.mark.parametrize( - "workers_values", - [ - { - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": {"labels": {"test_label": "test_label_value"}}, - }, - { - "labels": {"key": "value"}, - "celery": {"labels": {"test_label": "test_label_value"}}, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, }, show_only=["templates/workers/worker-service.yaml"], ) @@ -2785,17 +1866,9 @@ def test_should_add_component_specific_labels(self, workers_values): class TestWorkerCeleryServiceAccount: - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": False}}, - {"celery": {"serviceAccount": {"create": False}}}, - {"serviceAccount": {"create": True}, "celery": {"serviceAccount": {"create": False}}}, - ], - ) - def test_should_not_create_service_account_when_disabled(self, workers_values): + def test_should_not_create_service_account_when_disabled(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"serviceAccount": {"create": False}}}}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -2808,19 +1881,11 @@ def test_should_create_service_account_by_default(self): assert len(docs) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True}}, - {"celery": {"serviceAccount": {"create": True}}}, - {"serviceAccount": {"create": False}, "celery": {"serviceAccount": {"create": True}}}, - ], - ) - def test_should_not_create_service_account_for_local_executor(self, workers_values): + def test_should_not_create_service_account_for_local_executor(self): docs = render_chart( values={ "executor": "LocalExecutor", - "workers": workers_values, + "workers": {"celery": {"serviceAccount": {"create": True}}}, }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -2835,19 +1900,11 @@ def test_should_not_create_service_account_for_local_executor(self, workers_valu "KubernetesExecutor", ], ) - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True}}, - {"celery": {"serviceAccount": {"create": True}}}, - {"serviceAccount": {"create": False}, "celery": {"serviceAccount": {"create": True}}}, - ], - ) - def test_should_create_service_account_for_specific_executors(self, executor, workers_values): + def test_should_create_service_account_for_specific_executors(self, executor): docs = render_chart( values={ "executor": executor, - "workers": workers_values, + "workers": {"celery": {"serviceAccount": {"create": True}}}, }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -2855,89 +1912,43 @@ def test_should_create_service_account_for_specific_executors(self, executor, wo assert len(docs) == 1 assert jmespath.search("kind", docs[0]) == "ServiceAccount" - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True}}, - {"celery": {"serviceAccount": {"create": True}}}, - { - "serviceAccount": {"automountServiceAccountToken": False}, - "celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}, - }, - ], - ) - def test_automount_service_account_token_true(self, workers_values): + def test_automount_service_account_token_true(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"serviceAccount": {"create": True}}}}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is True - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, - {"celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}}, - { - "serviceAccount": {"automountServiceAccountToken": True}, - "celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, - }, - ], - ) - def test_automount_service_account_token_false(self, workers_values): + def test_automount_service_account_token_false(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}} + } + }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is False - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True, "name": "test"}}, - {"celery": {"serviceAccount": {"create": True, "name": "test"}}}, - { - "serviceAccount": {"name": "none"}, - "celery": {"serviceAccount": {"create": True, "name": "test"}}, - }, - ], - ) - def test_overwrite_name(self, workers_values): + def test_overwrite_name(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"serviceAccount": {"create": True, "name": "test"}}}}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) assert jmespath.search("metadata.name", docs[0]) == "test" - @pytest.mark.parametrize( - "workers_values", - [ - { - "celery": {"serviceAccount": {"create": True}}, - "labels": {"test_label": "test_label_value"}, - }, - { - "celery": { - "serviceAccount": {"create": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - { - "labels": {"key": "value"}, - "celery": { - "serviceAccount": {"create": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -2955,17 +1966,9 @@ def test_should_not_create_service_account_by_default(self): assert len(docs) == 0 - @pytest.mark.parametrize( - "workers_values", - [ - {"serviceAccount": {"create": True}}, # Should not have effect - {"kubernetes": {"serviceAccount": {"create": False}}}, - {"serviceAccount": {"create": True}, "kubernetes": {"serviceAccount": {"create": False}}}, - ], - ) - def test_should_not_create_service_account_when_disabled(self, workers_values): + def test_should_not_create_service_account_when_disabled(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"kubernetes": {"serviceAccount": {"create": False}}}}, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) @@ -3018,10 +2021,6 @@ def test_should_create_service_account_when_k8s_executors(self, executor): [ {"kubernetes": {"serviceAccount": {"create": True}}}, {"kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}}, - { - "serviceAccount": {"automountServiceAccountToken": False}, - "kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}, - }, ], ) def test_automount_service_account_token_true(self, workers_values): @@ -3031,85 +2030,52 @@ def test_automount_service_account_token_true(self, workers_values): ) assert jmespath.search("automountServiceAccountToken", docs[0]) is True - @pytest.mark.parametrize( - "workers_values", - [ - {"kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}}, - { - "serviceAccount": {"automountServiceAccountToken": True}, - "kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, - }, - ], - ) - def test_automount_service_account_token_false(self, workers_values): + def test_automount_service_account_token_false(self): docs = render_chart( - values={"executor": "KubernetesExecutor", "workers": workers_values}, + values={ + "executor": "KubernetesExecutor", + "workers": { + "kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}} + }, + }, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is False - @pytest.mark.parametrize( - "workers_values", - [ - {"kubernetes": {"serviceAccount": {"create": True}}}, - {"serviceAccount": {"name": "test"}, "kubernetes": {"serviceAccount": {"create": True}}}, - ], - ) - def test_default_name(self, workers_values): + def test_default_name(self): docs = render_chart( name="test", - values={"executor": "KubernetesExecutor", "workers": workers_values}, + values={ + "executor": "KubernetesExecutor", + "workers": {"kubernetes": {"serviceAccount": {"create": True}}}, + }, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) assert jmespath.search("metadata.name", docs[0]) == "test-airflow-worker-kubernetes" - @pytest.mark.parametrize( - "workers_values", - [ - {"kubernetes": {"serviceAccount": {"create": True, "name": "test"}}}, - { - "serviceAccount": {"name": "none"}, - "kubernetes": {"serviceAccount": {"create": True, "name": "test"}}, - }, - ], - ) - def test_overwrite_name(self, workers_values): + def test_overwrite_name(self): docs = render_chart( - values={"executor": "KubernetesExecutor", "workers": workers_values}, + values={ + "executor": "KubernetesExecutor", + "workers": {"kubernetes": {"serviceAccount": {"create": True, "name": "test"}}}, + }, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) assert jmespath.search("metadata.name", docs[0]) == "test" - @pytest.mark.parametrize( - "workers_values", - [ - { - "kubernetes": {"serviceAccount": {"create": True}}, - "labels": {"test_label": "test_label_value"}, - }, - { - "kubernetes": { - "serviceAccount": {"create": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - { - "labels": {"key": "value"}, - "kubernetes": { - "serviceAccount": {"create": True}, - "labels": {"test_label": "test_label_value"}, - }, - }, - ], - ) - def test_should_add_component_specific_labels(self, workers_values): + def test_should_add_component_specific_labels(self): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": workers_values, + "workers": { + "kubernetes": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, }, show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) diff --git a/chart/tests/helm_tests/airflow_core/test_worker_sets.py b/chart/tests/helm_tests/airflow_core/test_worker_sets.py index 3a3c035ec8d73..f54ffc6af41a5 100644 --- a/chart/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/chart/tests/helm_tests/airflow_core/test_worker_sets.py @@ -67,15 +67,6 @@ def test_create_multiple_worker_sets(self, enable_default, expected): "values", [ {"celery": {"enableDefault": False, "sets": [{"name": "set1", "replicas": 3}]}}, - {"replicas": 2, "celery": {"enableDefault": False, "sets": [{"name": "set1", "replicas": 3}]}}, - { - "replicas": 2, - "celery": { - "enableDefault": False, - "replicas": None, - "sets": [{"name": "set1", "replicas": 3}], - }, - }, {"celery": {"enableDefault": False, "replicas": None, "sets": [{"name": "set1", "replicas": 3}]}}, {"celery": {"enableDefault": False, "replicas": 2, "sets": [{"name": "set1", "replicas": 3}]}}, ], @@ -92,10 +83,6 @@ def test_overwrite_replicas(self, values): "values", [ {"celery": {"enableDefault": False, "sets": [{"name": "set1", "revisionHistoryLimit": 3}]}}, - { - "revisionHistoryLimit": 2, - "celery": {"enableDefault": False, "sets": [{"name": "set1", "revisionHistoryLimit": 3}]}, - }, { "celery": { "enableDefault": False, @@ -117,10 +104,6 @@ def test_overwrite_revision_history_limit(self, values): "values", [ {"celery": {"enableDefault": False, "sets": [{"name": "set1", "command": ["test"]}]}}, - { - "command": ["t"], - "celery": {"enableDefault": False, "sets": [{"name": "set1", "command": ["test"]}]}, - }, { "celery": { "enableDefault": False, @@ -144,15 +127,6 @@ def test_overwrite_command(self, values): "values", [ {"celery": {"enableDefault": False, "sets": [{"name": "set1", "args": ["test"]}]}}, - {"args": ["t"], "celery": {"enableDefault": False, "sets": [{"name": "set1", "args": ["test"]}]}}, - { - "args": ["t"], - "celery": { - "enableDefault": False, - "args": None, - "sets": [{"name": "set1", "args": ["test"]}], - }, - }, {"celery": {"enableDefault": False, "args": None, "sets": [{"name": "set1", "args": ["test"]}]}}, {"celery": {"enableDefault": False, "args": ["t"], "sets": [{"name": "set1", "args": ["test"]}]}}, ], @@ -185,29 +159,17 @@ def test_disable_livenessprobe(self): is None ) - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "livenessProbe": {"enabled": False}, - "sets": [{"name": "set1", "livenessProbe": {"enabled": True}}], + def test_overwrite_livenessprobe_enabled(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "livenessProbe": {"enabled": False}, + "sets": [{"name": "set1", "livenessProbe": {"enabled": True}}], + } } }, - { - "livenessProbe": {"enabled": False}, - "celery": { - "enableDefault": False, - "livenessProbe": {"enabled": None}, - "sets": [{"name": "set1", "livenessProbe": {"enabled": True}}], - }, - }, - ], - ) - def test_overwrite_livenessprobe_enabled(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -216,92 +178,34 @@ def test_overwrite_livenessprobe_enabled(self, values): is not None ) - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "livenessProbe": { - "initialDelaySeconds": 1, - "timeoutSeconds": 2, - "failureThreshold": 3, - "periodSeconds": 4, - "command": ["test"], - }, - "sets": [ - { - "name": "set1", - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "test"], - }, - } - ], + def test_overwrite_livenessprobe_values(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "livenessProbe": { + "initialDelaySeconds": 1, + "timeoutSeconds": 2, + "failureThreshold": 3, + "periodSeconds": 4, + "command": ["test"], + }, + "sets": [ + { + "name": "set1", + "livenessProbe": { + "initialDelaySeconds": 111, + "timeoutSeconds": 222, + "failureThreshold": 333, + "periodSeconds": 444, + "command": ["sh", "-c", "echo", "test"], + }, + } + ], + } } }, - { - "livenessProbe": { - "initialDelaySeconds": 1, - "timeoutSeconds": 2, - "failureThreshold": 3, - "periodSeconds": 4, - "command": ["test"], - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "test"], - }, - } - ], - }, - }, - { - "livenessProbe": { - "initialDelaySeconds": 1, - "timeoutSeconds": 2, - "failureThreshold": 3, - "periodSeconds": 4, - "command": ["test"], - }, - "celery": { - "enableDefault": False, - "livenessProbe": { - "initialDelaySeconds": None, - "timeoutSeconds": None, - "failureThreshold": None, - "periodSeconds": None, - }, - "sets": [ - { - "name": "set1", - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "test"], - }, - } - ], - }, - }, - ], - ) - def test_overwrite_livenessprobe_values(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -326,13 +230,6 @@ def test_overwrite_livenessprobe_values(self, values): "sets": [{"name": "set1", "updateStrategy": {"rollingUpdate": {"partition": 0}}}], } }, - { - "updateStrategy": {"rollingUpdate": {"partition": 1}}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "updateStrategy": {"rollingUpdate": {"partition": 0}}}], - }, - }, { "celery": { "enableDefault": False, @@ -365,33 +262,6 @@ def test_overwrite_update_strategy(self, values): ], }, }, - { - "strategy": {"rollingUpdate": {"maxSurge": "10%", "maxUnavailable": "90%"}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "persistence": {"enabled": False}, - "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - } - ], - }, - }, - { - "strategy": {"rollingUpdate": {"maxSurge": "10%", "maxUnavailable": "90%"}}, - "celery": { - "enableDefault": False, - "strategy": None, - "sets": [ - { - "name": "set1", - "persistence": {"enabled": False}, - "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -439,13 +309,6 @@ def test_overwrite_strategy(self, values): "sets": [{"name": "set1", "podManagementPolicy": "Parallel"}], } }, - { - "podManagementPolicy": "OrderedReady", - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "podManagementPolicy": "Parallel"}], - }, - }, { "celery": { "enableDefault": False, @@ -478,29 +341,17 @@ def test_disable_persistence(self): assert jmespath.search("kind", docs[0]) == "Deployment" - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "persistence": {"enabled": False}, - "sets": [{"name": "set1", "persistence": {"enabled": True}}], + def test_overwrite_persistence_enabled(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "persistence": {"enabled": False}, + "sets": [{"name": "set1", "persistence": {"enabled": True}}], + } } }, - { - "persistence": {"enabled": False}, - "celery": { - "enableDefault": False, - "persistence": {"enabled": None}, - "sets": [{"name": "set1", "persistence": {"enabled": True}}], - }, - }, - ], - ) - def test_overwrite_persistence_enabled(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -522,20 +373,6 @@ def test_overwrite_persistence_enabled(self, values): ], } }, - { - "persistence": {"persistentVolumeClaimRetentionPolicy": {"whenScaled": "Delete"}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "persistence": { - "persistentVolumeClaimRetentionPolicy": {"whenDeleted": "Delete"} - }, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -566,21 +403,6 @@ def test_overwrite_persistence_persistent_volume_claim_retention_policy(self, va "values", [ {"celery": {"enableDefault": False, "sets": [{"name": "set1", "persistence": {"size": "10Gi"}}]}}, - { - "persistence": {"size": "1Gi"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "persistence": {"size": "10Gi"}}], - }, - }, - { - "persistence": {"size": "1Gi"}, - "celery": { - "enableDefault": False, - "persistence": {"size": None}, - "sets": [{"name": "set1", "persistence": {"size": "10Gi"}}], - }, - }, { "celery": { "enableDefault": False, @@ -621,18 +443,6 @@ def test_overwrite_persistence_size(self, values): ], } }, - { - "persistence": {"storageClassName": "t"}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -676,29 +486,17 @@ def test_enable_persistence_fix_permissions(self): == 1 ) - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "persistence": {"fixPermissions": False}, - "sets": [{"name": "set1", "persistence": {"fixPermissions": True}}], + def test_overwrite_persistence_fix_permissions(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "persistence": {"fixPermissions": False}, + "sets": [{"name": "set1", "persistence": {"fixPermissions": True}}], + } } }, - { - "persistence": {"fixPermissions": False}, - "celery": { - "enableDefault": False, - "persistence": {"fixPermissions": None}, - "sets": [{"name": "set1", "persistence": {"fixPermissions": True}}], - }, - }, - ], - ) - def test_overwrite_persistence_fix_permissions(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -716,13 +514,6 @@ def test_overwrite_persistence_fix_permissions(self, values): "sets": [{"name": "set1", "persistence": {"annotations": {"foo": "bar"}}}], } }, - { - "persistence": {"annotations": {"a": "b"}}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "persistence": {"annotations": {"foo": "bar"}}}], - }, - }, { "celery": { "enableDefault": False, @@ -757,28 +548,17 @@ def test_overwrite_kerberos_init_container_enabled(self): jmespath.search("spec.template.spec.initContainers[?name=='kerberos-init']", docs[0]) is not None ) - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "kerberosInitContainer": {"enabled": True}, - "sets": [{"name": "test", "kerberosInitContainer": {"enabled": False}}], + def test_overwrite_kerberos_init_container_disable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "kerberosInitContainer": {"enabled": True}, + "sets": [{"name": "test", "kerberosInitContainer": {"enabled": False}}], + } } }, - { - "kerberosInitContainer": {"enabled": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "kerberosInitContainer": {"enabled": False}}], - }, - }, - ], - ) - def test_overwrite_kerberos_init_container_disable(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -787,51 +567,26 @@ def test_overwrite_kerberos_init_container_disable(self, values): is None ) - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosInitContainer": { - "enabled": True, - "resources": { - "limits": {"cpu": "3m", "memory": "4Mi"}, + def test_overwrite_kerberos_init_container_resources(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "sets": [ + { + "name": "test", + "kerberosInitContainer": { + "enabled": True, + "resources": { + "limits": {"cpu": "3m", "memory": "4Mi"}, + }, }, - }, - } - ], - } - }, - { - "kerberosInitContainer": { - "resources": { - "requests": {"cpu": "10m", "memory": "20Mi"}, + } + ], } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosInitContainer": { - "enabled": True, - "resources": { - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - }, - } - ], - }, + } }, - ], - ) - def test_overwrite_kerberos_init_container_resources(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -858,25 +613,6 @@ def test_overwrite_kerberos_init_container_resources(self, values): ], } }, - { - "kerberosInitContainer": { - "securityContexts": { - "container": {"allowPrivilegeEscalation": False}, - } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 10}}, - }, - } - ], - }, - }, { "celery": { "kerberosInitContainer": { @@ -927,25 +663,6 @@ def test_overwrite_kerberos_init_container_security_context(self, workers_values ], } }, - { - "kerberosInitContainer": { - "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "test"]}}} - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosInitContainer": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}}, - }, - }, - } - ], - }, - }, { "celery": { "kerberosInitContainer": { @@ -993,20 +710,6 @@ def test_overwrite_kerberos_init_container_lifecycle_hooks(self, workers_values) ], }, }, - { - "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "test"]}}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}} - }, - } - ], - }, - }, { "celery": { "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "test"]}}}, @@ -1097,13 +800,6 @@ def test_overwrite_pod_disruption_budget_enabled(self): "sets": [{"name": "test", "podDisruptionBudget": {"enabled": False}}], }, }, - { - "podDisruptionBudget": {"enabled": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "podDisruptionBudget": {"enabled": False}}], - }, - }, { "celery": { "enableDefault": False, @@ -1135,18 +831,6 @@ def test_overwrite_pod_disruption_budget_disable(self, workers_values): ], }, }, - { - "podDisruptionBudget": {"enabled": True, "config": {"maxUnavailable": 1}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "podDisruptionBudget": {"enabled": True, "config": {"minAvailable": 1}}, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -1221,13 +905,6 @@ def test_create_service_account_sets(self, enable_default, expected): "sets": [{"name": "test", "serviceAccount": {"create": False}}], } }, - { - "serviceAccount": {"create": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"create": False}}], - }, - }, ], ) def test_overwrite_service_account_create_disable(self, workers_values): @@ -1254,13 +931,6 @@ def test_overwrite_service_account_create_disable(self, workers_values): "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], } }, - { - "serviceAccount": {"automountServiceAccountToken": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], - }, - }, ], ) def test_overwrite_service_account_automount_service_account_token_disable(self, workers_values): @@ -1287,13 +957,6 @@ def test_overwrite_service_account_automount_service_account_token_disable(self, "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], } }, - { - "serviceAccount": {"name": "nontest"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], - }, - }, ], ) def test_overwrite_service_account_name(self, workers_values): @@ -1304,32 +967,17 @@ def test_overwrite_service_account_name(self, workers_values): assert jmespath.search("metadata.name", docs[0]) == "test" - @pytest.mark.parametrize( - "workers_values", - [ - { - "serviceAccount": {"annotations": {"test": "echo"}}, - "celery": {"enableDefault": False, "sets": [{"name": "test"}]}, - }, - { - "serviceAccount": {"annotations": {"echo": "test"}}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"annotations": {"test": "echo"}}}], - }, - }, - { - "celery": { - "enableDefault": False, - "serviceAccount": {"annotations": {"echo": "test"}}, - "sets": [{"name": "test", "serviceAccount": {"annotations": {"test": "echo"}}}], - }, - }, - ], - ) - def test_overwrite_service_account_annotations(self, workers_values): + def test_overwrite_service_account_annotations(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": { + "celery": { + "enableDefault": False, + "serviceAccount": {"annotations": {"echo": "test"}}, + "sets": [{"name": "test", "serviceAccount": {"annotations": {"test": "echo"}}}], + }, + } + }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -1338,7 +986,7 @@ def test_overwrite_service_account_annotations(self, workers_values): @pytest.mark.parametrize(("enable_default", "objects_number"), [(True, 1), (False, 0)]) def test_enable_default_keda(self, enable_default, objects_number): docs = render_chart( - values={"workers": {"celery": {"enableDefault": enable_default}, "keda": {"enabled": True}}}, + values={"workers": {"celery": {"enableDefault": enable_default, "keda": {"enabled": True}}}}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -1383,133 +1031,81 @@ def test_overwrite_keda_enabled(self): assert len(docs) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"enabled": True}, - "celery": {"enableDefault": False, "sets": [{"name": "test", "keda": {"enabled": False}}]}, - }, - { - "celery": { - "keda": {"enabled": True}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": False}}], + def test_overwrite_keda_disable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"enabled": True}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": False}}], + } } }, - ], - ) - def test_overwrite_keda_disable(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert len(docs) == 0 - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"pollingInterval": 1}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "pollingInterval": 10}}], - }, - }, - { - "celery": { - "keda": {"pollingInterval": 1}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "pollingInterval": 10}}], + def test_overwrite_keda_pooling_interval(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"pollingInterval": 1}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "pollingInterval": 10}}], + } } }, - ], - ) - def test_overwrite_keda_pooling_interval(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.pollingInterval", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"cooldownPeriod": 1}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "cooldownPeriod": 10}}], - }, - }, - { - "celery": { - "keda": {"cooldownPeriod": 1}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "cooldownPeriod": 10}}], + def test_overwrite_keda_cooldown_period(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"cooldownPeriod": 1}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "cooldownPeriod": 10}}], + } } }, - ], - ) - def test_overwrite_keda_cooldown_period(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.cooldownPeriod", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"minReplicaCount": 1}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "minReplicaCount": 10}}], - }, - }, - { - "celery": { - "keda": {"minReplicaCount": 1}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "minReplicaCount": 10}}], + def test_overwrite_keda_min_replica_count(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"minReplicaCount": 1}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "minReplicaCount": 10}}], + } } }, - ], - ) - def test_overwrite_keda_min_replica_count(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.minReplicaCount", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"maxReplicaCount": 1}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "maxReplicaCount": 5}}], - }, - }, - { - "celery": { - "keda": {"maxReplicaCount": 1}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "maxReplicaCount": 5}}], + def test_overwrite_keda_max_replica_count(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"maxReplicaCount": 1}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "maxReplicaCount": 5}}], + } } }, - ], - ) - def test_overwrite_keda_max_replica_count(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -1537,47 +1133,18 @@ def test_overwrite_keda_max_replica_count(self, workers_values): } }, { - "keda": { - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": { - "scaleDown": { - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}] + "celery": { + "keda": { + "advanced": { + "horizontalPodAutoscalerConfig": { + "behavior": { + "scaleDown": { + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}] + } } } } - } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "keda": { - "enabled": True, - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": {"scaleDown": {"stabilizationWindowSeconds": 300}} - } - }, - }, - } - ], - }, - }, - { - "celery": { - "keda": { - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": { - "scaleDown": { - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}] - } - } - } - } - }, + }, "enableDefault": False, "sets": [ { @@ -1606,28 +1173,17 @@ def test_overwrite_keda_advanced(self, workers_values): "horizontalPodAutoscalerConfig": {"behavior": {"scaleDown": {"stabilizationWindowSeconds": 300}}} } - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"query": "not"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "query": "test"}}], - }, - }, - { - "celery": { - "keda": {"query": "not"}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "query": "test"}}], + def test_overwrite_keda_query(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": {"query": "not"}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "query": "test"}}], + } } }, - ], - ) - def test_overwrite_keda_query(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -1691,30 +1247,17 @@ def test_overwrite_queue_keda_query_with_default(self, queue, expected_in_query) assert "queue IN ('default')" in default_query assert expected_in_query not in default_query - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"usePgbouncer": False}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": True}}], - }, - }, - { - "celery": { - "keda": {"usePgbouncer": False}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": True}}], - } - }, - ], - ) - def test_overwrite_keda_use_pgbouncer_enable(self, workers_values): + def test_overwrite_keda_use_pgbouncer_enable(self): docs = render_chart( values={ "pgbouncer": {"enabled": True}, - "workers": workers_values, + "workers": { + "celery": { + "keda": {"usePgbouncer": False}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": True}}], + } + }, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -1724,30 +1267,17 @@ def test_overwrite_keda_use_pgbouncer_enable(self, workers_values): == "AIRFLOW_CONN_AIRFLOW_DB" ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": {"usePgbouncer": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": False}}], - }, - }, - { - "celery": { - "keda": {"usePgbouncer": True}, - "enableDefault": False, - "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": False}}], - } - }, - ], - ) - def test_overwrite_keda_use_pgbouncer_disable(self, workers_values): + def test_overwrite_keda_use_pgbouncer_disable(self): docs = render_chart( values={ "pgbouncer": {"enabled": True}, - "workers": workers_values, + "workers": { + "celery": { + "keda": {"usePgbouncer": True}, + "enableDefault": False, + "sets": [{"name": "test", "keda": {"enabled": True, "usePgbouncer": False}}], + } + }, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) @@ -1774,7 +1304,7 @@ def test_overwrite_queue(self): @pytest.mark.parametrize(("enable_default", "objects_number"), [(True, 1), (False, 0)]) def test_enable_default_hpa(self, enable_default, objects_number): docs = render_chart( - values={"workers": {"celery": {"enableDefault": enable_default}, "hpa": {"enabled": True}}}, + values={"workers": {"celery": {"enableDefault": enable_default, "hpa": {"enabled": True}}}}, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -1818,10 +1348,6 @@ def test_create_hpa_sets(self, enable_default, expected): "sets": [{"name": "test", "hpa": {"enabled": True}}], } }, - { - "hpa": {"enabled": False}, - "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": True}}]}, - }, ], ) def test_overwrite_hpa_enabled(self, workers_values): @@ -1832,25 +1358,17 @@ def test_overwrite_hpa_enabled(self, workers_values): assert len(docs) == 1 - @pytest.mark.parametrize( - "workers_values", - [ - { - "celery": { - "enableDefault": False, - "hpa": {"enabled": True}, - "sets": [{"name": "test", "hpa": {"enabled": False}}], + def test_overwrite_hpa_disable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "hpa": {"enabled": True}, + "sets": [{"name": "test", "hpa": {"enabled": False}}], + } } }, - { - "hpa": {"enabled": True}, - "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": False}}]}, - }, - ], - ) - def test_overwrite_hpa_disable(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -1872,13 +1390,6 @@ def test_overwrite_hpa_disable(self, workers_values): "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], } }, - { - "hpa": {"minReplicaCount": 7}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], - }, - }, ], ) def test_overwrite_hpa_min_replica_count(self, workers_values): @@ -1905,13 +1416,6 @@ def test_overwrite_hpa_min_replica_count(self, workers_values): "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], } }, - { - "hpa": {"maxReplicaCount": 7}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], - }, - }, ], ) def test_overwrite_hpa_max_replica_count(self, workers_values): @@ -1947,39 +1451,6 @@ def test_overwrite_hpa_max_replica_count(self, workers_values): ], } }, - { - "hpa": { - "metrics": [ - { - "type": "Resource", - "resource": { - "name": "memory", - "target": {"type": "Utilization", "averageUtilization": 1}, - }, - } - ], - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "hpa": { - "enabled": True, - "metrics": [ - { - "type": "Resource", - "resource": { - "name": "cpu", - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - } - ], - }, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -2042,18 +1513,6 @@ def test_overwrite_hpa_metrics(self, workers_values): ], } }, - { - "hpa": {"behavior": {"scaleUp": {"selectPolicy": "Min"}}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "hpa": {"enabled": True, "behavior": {"scaleDown": {"selectPolicy": "Max"}}}, - } - ], - }, - }, { "celery": { "enableDefault": False, @@ -2095,28 +1554,17 @@ def test_overwrite_kerberos_sidecar_enabled(self, workers_celery_values): assert jmespath.search("spec.template.spec.containers[?name=='worker-kerberos']", docs[0]) is not None - @pytest.mark.parametrize( - "values", - [ - { - "celery": { - "enableDefault": False, - "kerberosSidecar": {"enabled": True}, - "sets": [{"name": "test", "kerberosSidecar": {"enabled": False}}], + def test_overwrite_kerberos_sidecar_disable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "kerberosSidecar": {"enabled": True}, + "sets": [{"name": "test", "kerberosSidecar": {"enabled": False}}], + } } }, - { - "kerberosSidecar": {"enabled": True}, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "kerberosSidecar": {"enabled": False}}], - }, - }, - ], - ) - def test_overwrite_kerberos_sidecar_disable(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2143,27 +1591,6 @@ def test_overwrite_kerberos_sidecar_disable(self, values): ], } }, - { - "kerberosSidecar": { - "resources": { - "requests": {"cpu": "10m", "memory": "20Mi"}, - } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosSidecar": { - "enabled": True, - "resources": { - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - }, - } - ], - }, - }, { "celery": { "kerberosSidecar": { @@ -2218,27 +1645,6 @@ def test_overwrite_kerberos_sidecar_resources(self, values): ], } }, - { - "kerberosSidecar": { - "securityContexts": { - "container": {"allowPrivilegeEscalation": False}, - } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosSidecar": { - "enabled": True, - "securityContexts": { - "container": {"runAsUser": 10}, - }, - }, - } - ], - }, - }, { "celery": { "kerberosSidecar": { @@ -2291,25 +1697,6 @@ def test_overwrite_kerberos_sidecar_security_context_container(self, values): ], } }, - { - "kerberosSidecar": { - "containerLifecycleHooks": {"preStop": {"exec": {"command": ["echo", "test"]}}} - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "kerberosSidecar": { - "enabled": True, - "containerLifecycleHooks": { - "postStart": {"exec": {"command": ["echo", "{{ .Release.Name }}"]}}, - }, - }, - } - ], - }, - }, { "celery": { "kerberosSidecar": { @@ -2357,22 +1744,6 @@ def test_overwrite_kerberos_sidecar_container_lifecycle_hooks(self, values): ], } }, - { - "resources": { - "requests": {"cpu": "10m", "memory": "20Mi"}, - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "resources": { - "limits": {"cpu": "3m", "memory": "4Mi"}, - }, - } - ], - }, - }, { "celery": { "resources": { @@ -2410,13 +1781,6 @@ def test_overwrite_resources(self, values): "sets": [{"name": "test", "terminationGracePeriodSeconds": 5}], } }, - { - "terminationGracePeriodSeconds": 20, - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "terminationGracePeriodSeconds": 5}], - }, - }, { "celery": { "terminationGracePeriodSeconds": 20, @@ -2445,10 +1809,6 @@ def test_overwrite_termination_grace_period_seconds(self, workers_values): "sets": [{"name": "set1", "safeToEvict": True}], } }, - { - "safeToEvict": False, - "celery": {"enableDefault": False, "sets": [{"name": "set1", "safeToEvict": True}]}, - }, ], ) def test_overwrite_safe_to_evict_enable(self, workers_values): @@ -2464,25 +1824,17 @@ def test_overwrite_safe_to_evict_enable(self, workers_values): == "true" ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "celery": { - "safeToEvict": True, - "enableDefault": False, - "sets": [{"name": "set1", "safeToEvict": False}], + def test_overwrite_safe_to_evict_disable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "safeToEvict": True, + "enableDefault": False, + "sets": [{"name": "set1", "safeToEvict": False}], + } } }, - { - "safeToEvict": True, - "celery": {"enableDefault": False, "sets": [{"name": "set1", "safeToEvict": False}]}, - }, - ], - ) - def test_overwrite_safe_to_evict_disable(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2493,49 +1845,23 @@ def test_overwrite_safe_to_evict_disable(self, workers_values): == "false" ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "extraContainers": [{"name": "test", "image": "test"}], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "extraContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "extraContainers": [{"name": "test", "image": "test"}], - "sets": [ - { - "name": "set1", - "extraContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - } - ], - }, - }, - ], - ) - def test_overwrite_extra_containers(self, workers_values): + def test_overwrite_extra_containers(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "extraContainers": [{"name": "test", "image": "test"}], + "sets": [ + { + "name": "set1", + "extraContainers": [ + {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2548,49 +1874,23 @@ def test_overwrite_extra_containers(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraInitContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "extraInitContainers": [{"name": "test", "image": "test"}], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "extraInitContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "extraInitContainers": [{"name": "test", "image": "test"}], - "sets": [ - { - "name": "set1", - "extraInitContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - } - ], - }, - }, - ], - ) - def test_overwrite_extra_init_containers(self, workers_values): + def test_overwrite_extra_init_containers(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "extraInitContainers": [{"name": "test", "image": "test"}], + "sets": [ + { + "name": "set1", + "extraInitContainers": [ + {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2603,43 +1903,21 @@ def test_overwrite_extra_init_containers(self, workers_values): "image": "test-registry/test-repo:test-tag", } - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "extraVolumes": [{"name": "test", "emptyDir": {}}], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "extraVolumes": [{"name": "test", "emptyDir": {}}], - "sets": [ - { - "name": "set1", - "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], - } - ], - }, - }, - ], - ) - def test_overwrite_extra_volumes(self, workers_values): + def test_overwrite_extra_volumes(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "extraVolumes": [{"name": "test", "emptyDir": {}}], + "sets": [ + { + "name": "set1", + "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2649,49 +1927,23 @@ def test_overwrite_extra_volumes(self, workers_values): "emptyDir": {}, } - @pytest.mark.parametrize( - "workers_values", - [ - { - "extraVolumeMounts": [ - {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "extraVolumeMounts": [ - {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], - "sets": [ - { - "name": "set1", - "extraVolumeMounts": [ - {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} - ], - } - ], - }, - }, - ], - ) - def test_overwrite_extra_volume_mounts(self, workers_values): + def test_overwrite_extra_volume_mounts(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], + "sets": [ + { + "name": "set1", + "extraVolumeMounts": [ + {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2704,10 +1956,6 @@ def test_overwrite_extra_volume_mounts(self, workers_values): @pytest.mark.parametrize( "workers_values", [ - { - "extraPorts": [{"name": "test-extra-port", "containerPort": 10}], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, { "celery": { "extraPorts": [{"name": "test-extra-port", "containerPort": 10}], @@ -2715,15 +1963,6 @@ def test_overwrite_extra_volume_mounts(self, workers_values): "sets": [{"name": "set1"}], }, }, - { - "extraPorts": [{"name": "test", "containerPort": 1}], - "celery": { - "enableDefault": False, - "sets": [ - {"name": "set1", "extraPorts": [{"name": "test-extra-port", "containerPort": 10}]} - ], - }, - }, { "celery": { "extraPorts": [{"name": "test", "containerPort": 1}], @@ -2747,206 +1986,99 @@ def test_overwrite_extra_ports(self, workers_values): {"name": "test-extra-port", "containerPort": 10} ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "nodeSelector": {"name": "test-node"}, - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "nodeSelector": {"test": "name"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "nodeSelector": {"name": "test-node"}}], - }, - }, - { - "celery": { - "nodeSelector": {"test": "name"}, - "enableDefault": False, - "sets": [{"name": "set1", "nodeSelector": {"name": "test-node"}}], - }, - }, - ], - ) - def test_overwrite_node_selector(self, workers_values): + def test_overwrite_node_selector(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "nodeSelector": {"test": "name"}, + "enableDefault": False, + "sets": [{"name": "set1", "nodeSelector": {"name": "test-node"}}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.spec.nodeSelector", docs[0]) == {"name": "test-node"} - @pytest.mark.parametrize( - "workers_values", - [ - { - "runtimeClassName": "test-class", - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "runtimeClassName": "test", - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "runtimeClassName": "test-class"}], - }, - }, - { - "celery": { - "runtimeClassName": "test", - "enableDefault": False, - "sets": [{"name": "set1", "runtimeClassName": "test-class"}], - }, - }, - ], - ) - def test_overwrite_runtime_class_name(self, workers_values): + def test_overwrite_runtime_class_name(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "runtimeClassName": "test", + "enableDefault": False, + "sets": [{"name": "set1", "runtimeClassName": "test-class"}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.spec.runtimeClassName", docs[0]) == "test-class" - @pytest.mark.parametrize( - "workers_values", - [ - { - "priorityClassName": "test-class", - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "priorityClassName": "test", - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "priorityClassName": "test-class"}], - }, - }, - { - "celery": { - "priorityClassName": "test", - "enableDefault": False, - "sets": [{"name": "set1", "priorityClassName": "test-class"}], - }, - }, - ], - ) - def test_overwrite_priority_class_name(self, workers_values): + def test_overwrite_priority_class_name(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "priorityClassName": "test", + "enableDefault": False, + "sets": [{"name": "set1", "priorityClassName": "test-class"}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.spec.priorityClassName", docs[0]) == "test-class" - @pytest.mark.parametrize( - "workers_values", - [ - { - "affinity": { - "nodeAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 1, - "preference": { - "matchExpressions": [ - {"key": "not-me", "operator": "In", "values": ["true"]}, - ] - }, + def test_overwrite_affinity(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "enableDefault": False, + "affinity": { + "podAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "topologyKey": "foo", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + }, + "weight": 1, + } + ] } - ] - } - }, - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "affinity": { - "podAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ + }, + "sets": [ { - "podAffinityTerm": { - "topologyKey": "foo", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, + "name": "set1", + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 1, + "preference": { + "matchExpressions": [ + { + "key": "not-me", + "operator": "In", + "values": ["true"], + }, + ] + }, + } + ] + } }, - "weight": 1, } - ] - } - }, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "affinity": { - "nodeAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 1, - "preference": { - "matchExpressions": [ - {"key": "not-me", "operator": "In", "values": ["true"]}, - ] - }, - } - ] - } - }, - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "affinity": { - "podAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "topologyKey": "foo", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - }, - "weight": 1, - } - ] - } + ], }, - "sets": [ - { - "name": "set1", - "affinity": { - "nodeAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 1, - "preference": { - "matchExpressions": [ - {"key": "not-me", "operator": "In", "values": ["true"]}, - ] - }, - } - ] - } - }, - } - ], }, }, - ], - ) - def test_overwrite_affinity(self, workers_values): - docs = render_chart( - values={ - "workers": workers_values, - }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -2965,63 +2097,30 @@ def test_overwrite_affinity(self, workers_values): } } - @pytest.mark.parametrize( - "workers_values", - [ - { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "tolerations": [ - {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "tolerations": [ - { - "key": "dynamic-pods", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule", - } - ], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "tolerations": [ - {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - "sets": [ - { - "name": "set1", - "tolerations": [ - { - "key": "dynamic-pods", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule", - } - ], - } - ], - }, - }, - ], - ) - def test_overwrite_tolerations(self, workers_values): + def test_overwrite_tolerations(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "tolerations": [ + {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + "sets": [ + { + "name": "set1", + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3030,78 +2129,35 @@ def test_overwrite_tolerations(self, workers_values): {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "not-me", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "not-me", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - "sets": [ - { - "name": "set1", - "topologySpreadConstraints": [ - { - "maxSkew": 1, - "topologyKey": "foo", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": {"matchLabels": {"tier": "airflow"}}, - } - ], - } - ], - }, - }, - ], - ) - def test_overwrite_topology_spread_constraints(self, workers_values): + def test_overwrite_topology_spread_constraints(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "not-me", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + "sets": [ + { + "name": "set1", + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3115,130 +2171,72 @@ def test_overwrite_topology_spread_constraints(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "hostAliases": [{"ip": "192.168.0.0", "hostnames": ["test"]}], - "celery": { - "enableDefault": False, - "sets": [ - {"name": "set1", "hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]} - ], - }, - }, - { - "celery": { - "hostAliases": [{"ip": "192.168.0.0", "hostnames": ["test"]}], - "enableDefault": False, - "sets": [ - {"name": "set1", "hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}]} - ], - }, - }, - ], - ) - def test_overwrite_host_aliases(self, workers_values): + def test_overwrite_host_aliases(self): docs = render_chart( values={ - "workers": workers_values, - }, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert jmespath.search("spec.template.spec.hostAliases", docs[0]) == [ - {"ip": "127.0.0.2", "hostnames": ["test.hostname"]} - ] - - @pytest.mark.parametrize( - "workers_values", - [ - {"annotations": {"test": "echo"}, "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}}, - { - "annotations": {"echo": "test"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "annotations": {"test": "echo"}}], - }, - }, - { - "celery": { - "enableDefault": False, - "annotations": {"echo": "test"}, - "sets": [{"name": "set1", "annotations": {"test": "echo"}}], + "workers": { + "celery": { + "hostAliases": [{"ip": "192.168.0.0", "hostnames": ["test"]}], + "enableDefault": False, + "sets": [ + { + "name": "set1", + "hostAliases": [{"ip": "127.0.0.2", "hostnames": ["test.hostname"]}], + } + ], + }, }, }, - ], - ) - def test_overwrite_annotations(self, workers_values): + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert jmespath.search("spec.template.spec.hostAliases", docs[0]) == [ + {"ip": "127.0.0.2", "hostnames": ["test.hostname"]} + ] + + def test_overwrite_annotations(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "annotations": {"echo": "test"}, + "sets": [{"name": "set1", "annotations": {"test": "echo"}}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("metadata.annotations", docs[0]) == {"test": "echo"} - @pytest.mark.parametrize( - "workers_values", - [ - { - "podAnnotations": {"test": "echo"}, - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "podAnnotations": {"echo": "test"}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "podAnnotations": {"test": "echo"}}], - }, - }, - { - "celery": { - "enableDefault": False, - "podAnnotations": {"echo": "test"}, - "sets": [{"name": "set1", "podAnnotations": {"test": "echo"}}], - }, - }, - ], - ) - def test_overwrite_pod_annotations(self, workers_values): + def test_overwrite_pod_annotations(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "podAnnotations": {"echo": "test"}, + "sets": [{"name": "set1", "podAnnotations": {"test": "echo"}}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("spec.template.metadata.annotations", docs[0])["test"] == "echo" - @pytest.mark.parametrize( - "workers_values", - [ - {"labels": {"test": "echo"}, "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}}, - { - "labels": {"echo": "test"}, - "celery": {"enableDefault": False, "sets": [{"name": "set1", "labels": {"test": "echo"}}]}, - }, - { - "celery": { - "enableDefault": False, - "labels": {"echo": "test"}, - "sets": [{"name": "set1", "labels": {"test": "echo"}}], - }, - }, - ], - ) - def test_overwrite_labels(self, workers_values): + def test_overwrite_labels(self): docs = render_chart( values={ "labels": {"global-test": "global-echo"}, - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "labels": {"echo": "test"}, + "sets": [{"name": "set1", "labels": {"test": "echo"}}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3279,28 +2277,17 @@ def test_overwrite_wait_for_migration_disable(self, workers_values): is None ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "waitForMigrations": {"enabled": False}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], - }, - }, - { - "celery": { - "waitForMigrations": {"enabled": False}, - "enableDefault": False, - "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], + def test_overwrite_wait_for_migration_enable(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "waitForMigrations": {"enabled": False}, + "enableDefault": False, + "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], + } } }, - ], - ) - def test_overwrite_wait_for_migration_enable(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) assert ( @@ -3310,43 +2297,21 @@ def test_overwrite_wait_for_migration_enable(self, workers_values): is not None ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, - } - ], - }, - }, - { - "celery": { - "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, - "enableDefault": False, - "sets": [ - { - "name": "set1", - "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, - } - ], - }, - }, - ], - ) - def test_overwrite_wait_for_migration_env(self, workers_values): + def test_overwrite_wait_for_migration_env(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, + "enableDefault": False, + "sets": [ + { + "name": "set1", + "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3358,45 +2323,23 @@ def test_overwrite_wait_for_migration_env(self, workers_values): assert {"name": "TEST_ENV_1", "value": "test_env_1"} in envs assert {"name": "TEST", "value": "test"} not in envs - @pytest.mark.parametrize( - "workers_values", - [ - { - "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 10}}}, - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "waitForMigrations": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 10}}}, - } - ], - }, - }, - { - "celery": { - "waitForMigrations": { - "securityContexts": {"container": {"allowPrivilegeEscalation": False}} - }, - "enableDefault": False, - "sets": [ - { - "name": "set1", - "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 10}}}, - } - ], - }, - }, - ], - ) - def test_overwrite_wait_for_migration_security_context_container(self, workers_values): + def test_overwrite_wait_for_migration_security_context_container(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + }, + "enableDefault": False, + "sets": [ + { + "name": "set1", + "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 10}}}, + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3406,33 +2349,16 @@ def test_overwrite_wait_for_migration_security_context_container(self, workers_v docs[0], ) == {"runAsUser": 10} - @pytest.mark.parametrize( - "workers_values", - [ - { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "env": [{"name": "TEST", "value": "test"}], - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}], - }, - }, - { - "celery": { - "enableDefault": False, - "env": [{"name": "TEST", "value": "test"}], - "sets": [{"name": "set1", "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}], - }, - }, - ], - ) - def test_overwrite_env(self, workers_values): + def test_overwrite_env(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "enableDefault": False, + "env": [{"name": "TEST", "value": "test"}], + "sets": [{"name": "set1", "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3442,88 +2368,39 @@ def test_overwrite_env(self, workers_values): assert {"name": "TEST_ENV_1", "value": "test_env_1"} in envs assert {"name": "TEST", "value": "test"} not in envs - @pytest.mark.parametrize( - "workers_values", - [ - { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - }, - } - ], - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume"}, - "spec": { - "storageClassName": "class", - "accessModes": ["ReadOnce"], - "resources": {"requests": {"storage": "1Gi"}}, - }, - } - ], - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - }, - } - ], - } - ], - }, - }, - { - "celery": { - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume"}, - "spec": { - "storageClassName": "class", - "accessModes": ["ReadOnce"], - "resources": {"requests": {"storage": "1Gi"}}, - }, - } - ], - "enableDefault": False, - "sets": [ - { - "name": "set1", - "volumeClaimTemplates": [ - { - "metadata": {"name": "test-volume-airflow-1"}, - "spec": { - "storageClassName": "storage-class-1", - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - }, - } - ], - } - ], - }, - }, - ], - ) - def test_overwrite_volume_claim_templates(self, workers_values): + def test_overwrite_volume_claim_templates(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "volumeClaimTemplates": [ + { + "metadata": {"name": "test-volume"}, + "spec": { + "storageClassName": "class", + "accessModes": ["ReadOnce"], + "resources": {"requests": {"storage": "1Gi"}}, + }, + } + ], + "enableDefault": False, + "sets": [ + { + "name": "set1", + "volumeClaimTemplates": [ + { + "metadata": {"name": "test-volume-airflow-1"}, + "spec": { + "storageClassName": "storage-class-1", + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + }, + } + ], + } + ], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -3539,33 +2416,16 @@ def test_overwrite_volume_claim_templates(self, workers_values): } ] - @pytest.mark.parametrize( - "workers_values", - [ - { - "schedulerName": "most-allocated", - "celery": {"enableDefault": False, "sets": [{"name": "set1"}]}, - }, - { - "schedulerName": "test", - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "schedulerName": "most-allocated"}], - }, - }, - { - "celery": { - "schedulerName": "test", - "enableDefault": False, - "sets": [{"name": "set1", "schedulerName": "most-allocated"}], - }, - }, - ], - ) - def test_overwrite_scheduler_name(self, workers_values): + def test_overwrite_scheduler_name(self): docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "schedulerName": "test", + "enableDefault": False, + "sets": [{"name": "set1", "schedulerName": "most-allocated"}], + }, + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) diff --git a/chart/tests/helm_tests/other/test_hpa.py b/chart/tests/helm_tests/other/test_hpa.py index 5452d40377453..41f4ace79070e 100644 --- a/chart/tests/helm_tests/other/test_hpa.py +++ b/chart/tests/helm_tests/other/test_hpa.py @@ -35,21 +35,10 @@ def test_hpa_disabled_by_default(self): "CeleryExecutor,KubernetesExecutor", ], ) - @pytest.mark.parametrize( - "workers_values", - [ - {"hpa": {"enabled": True}, "celery": {"persistence": {"enabled": False}}}, - {"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}}, - { - "hpa": {"enabled": False}, - "celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}, - }, - ], - ) - def test_hpa_enabled(self, executor, workers_values): + def test_hpa_enabled(self, executor): docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}}, "executor": executor, }, show_only=["templates/workers/worker-hpa.yaml"], @@ -66,20 +55,11 @@ def test_min_max_replicas_default(self): assert jmespath.search("spec.minReplicas", docs[0]) == 0 assert jmespath.search("spec.maxReplicas", docs[0]) == 5 - @pytest.mark.parametrize( - "workers_values", - [ - {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}, - {"celery": {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}}, - { - "hpa": {"enabled": True, "minReplicaCount": 1, "maxReplicaCount": 10}, - "celery": {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}, - }, - ], - ) - def test_min_max_replicas(self, workers_values): + def test_min_max_replicas(self): docs = render_chart( - values={"workers": workers_values}, + values={ + "workers": {"celery": {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}} + }, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -87,61 +67,23 @@ def test_min_max_replicas(self, workers_values): assert jmespath.search("spec.maxReplicas", docs[0]) == 8 @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryExecutor,KubernetesExecutor"]) - @pytest.mark.parametrize( - "workers_values", - [ - { - "hpa": { - "enabled": True, - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - }, - } - }, - { - "celery": { - "hpa": { - "enabled": True, - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - }, - } - } - }, - { - "hpa": { - "behavior": { - "scaleUp": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - } - }, - "celery": { - "hpa": { - "enabled": True, - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - }, - } - }, - }, - ], - ) - def test_hpa_behavior(self, executor, workers_values): + def test_hpa_behavior(self, executor): """Verify HPA behavior.""" docs = render_chart( values={ - "workers": workers_values, + "workers": { + "celery": { + "hpa": { + "enabled": True, + "behavior": { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + }, + } + } + }, "executor": executor, }, show_only=["templates/workers/worker-hpa.yaml"], @@ -158,8 +100,6 @@ def test_hpa_behavior(self, executor, workers_values): [ ({"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": True}}}, "StatefulSet"), ({"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}}, "Deployment"), - ({"persistence": {"enabled": True}, "celery": {"hpa": {"enabled": True}}}, "StatefulSet"), - ({"persistence": {"enabled": False}, "celery": {"hpa": {"enabled": True}}}, "Deployment"), ], ) def test_persistence(self, workers_values, kind): diff --git a/chart/tests/helm_tests/other/test_keda.py b/chart/tests/helm_tests/other/test_keda.py index 0c2e83a9e520c..41b07021e0a60 100644 --- a/chart/tests/helm_tests/other/test_keda.py +++ b/chart/tests/helm_tests/other/test_keda.py @@ -29,13 +29,6 @@ def test_keda_disabled_by_default(self): docs = render_chart(show_only=["templates/workers/worker-kedaautoscaler.yaml"]) assert docs == [] - def test_keda_disabled_overwrite(self): - docs = render_chart( - values={"workers": {"keda": {"enabled": True}, "celery": {"keda": {"enabled": False}}}}, - show_only=["templates/workers/worker-kedaautoscaler.yaml"], - ) - assert docs == [] - @pytest.mark.parametrize( "executor", [ @@ -43,18 +36,11 @@ def test_keda_disabled_overwrite(self): "CeleryExecutor,KubernetesExecutor", ], ) - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True}, "celery": {"persistence": {"enabled": False}}}, - {"celery": {"keda": {"enabled": True}, "persistence": {"enabled": False}}}, - ], - ) - def test_keda_enabled(self, executor, workers_values): + def test_keda_enabled(self, executor): """ScaledObject should only be created when enabled and executor is Celery or CeleryKubernetes.""" docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True}, "persistence": {"enabled": False}}}, "executor": executor, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], @@ -74,72 +60,29 @@ def test_include_event_source_container_name_in_scaled_object(self, executor): assert jmespath.search("spec.scaleTargetRef.envSourceContainerName", docs[0]) == "worker" @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryExecutor,KubernetesExecutor"]) - @pytest.mark.parametrize( - "workers_values", - [ - { - "keda": { - "enabled": True, - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - } - } - }, - } - }, - { - "celery": { - "keda": { - "enabled": True, - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + def test_keda_advanced(self, executor): + """Verify keda advanced config.""" + docs = render_chart( + values={ + "workers": { + "celery": { + "keda": { + "enabled": True, + "advanced": { + "horizontalPodAutoscalerConfig": { + "behavior": { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [ + {"type": "Percent", "value": 100, "periodSeconds": 15} + ], + } } } - } - }, - } - } - }, - { - "keda": { - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": {"scaleUp": {"stabilizationWindowSeconds": 5}} + }, } } }, - "celery": { - "keda": { - "enabled": True, - "advanced": { - "horizontalPodAutoscalerConfig": { - "behavior": { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - } - } - }, - } - }, - }, - ], - ) - def test_keda_advanced(self, executor, workers_values): - """Verify keda advanced config.""" - docs = render_chart( - values={ - "workers": workers_values, "executor": executor, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], @@ -233,8 +176,6 @@ def test_keda_query_kubernetes_queue(self, executor, queue): [ ({"celery": {"keda": {"enabled": True}, "persistence": {"enabled": True}}}, "StatefulSet"), ({"celery": {"keda": {"enabled": True}, "persistence": {"enabled": False}}}, "Deployment"), - ({"persistence": {"enabled": True}, "celery": {"keda": {"enabled": True}}}, "StatefulSet"), - ({"persistence": {"enabled": False}, "celery": {"keda": {"enabled": True}}}, "Deployment"), ], ) def test_persistence(self, workers_values, kind): @@ -249,20 +190,13 @@ def test_persistence(self, workers_values, kind): assert jmespath.search("spec.scaleTargetRef.kind", docs[0]) == kind - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}}}, - ], - ) - def test_default_keda_db_connection(self, workers_values): + def test_default_keda_db_connection(self): """Verify default keda db connection.""" import base64 docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True}}}, "executor": "CeleryExecutor", }, show_only=[ @@ -291,20 +225,13 @@ def test_default_keda_db_connection(self, workers_values): ) assert autoscaler_connection_env_var == "AIRFLOW_CONN_AIRFLOW_DB" - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}}}, - ], - ) - def test_default_keda_db_connection_pgbouncer_enabled(self, workers_values): + def test_default_keda_db_connection_pgbouncer_enabled(self): """Verify keda db connection when pgbouncer is enabled.""" import base64 docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True}}}, "executor": "CeleryExecutor", "pgbouncer": {"enabled": True}, }, @@ -334,20 +261,13 @@ def test_default_keda_db_connection_pgbouncer_enabled(self, workers_values): ) assert autoscaler_connection_env_var == "AIRFLOW_CONN_AIRFLOW_DB" - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True, "usePgbouncer": False}}, - {"celery": {"keda": {"enabled": True, "usePgbouncer": False}}}, - ], - ) - def test_default_keda_db_connection_pgbouncer_enabled_usePgbouncer_false(self, workers_values): + def test_default_keda_db_connection_pgbouncer_enabled_usePgbouncer_false(self): """Verify keda db connection when pgbouncer is enabled and usePgbouncer is false.""" import base64 docs = render_chart( values={ - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True, "usePgbouncer": False}}}, "executor": "CeleryExecutor", "pgbouncer": {"enabled": True}, }, @@ -384,21 +304,14 @@ def test_default_keda_db_connection_pgbouncer_enabled_usePgbouncer_false(self, w ) assert autoscaler_connection_env_var == "KEDA_DB_CONN" - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}}}, - ], - ) - def test_mysql_keda_db_connection(self, workers_values): + def test_mysql_keda_db_connection(self): """Verify keda db connection when pgbouncer is enabled.""" import base64 docs = render_chart( values={ "data": {"metadataConnection": {"protocol": "mysql", "port": 3306}}, - "workers": workers_values, + "workers": {"celery": {"keda": {"enabled": True}}}, "executor": "CeleryExecutor", }, show_only=[ @@ -462,65 +375,33 @@ def test_keda_query_queue_list(self, queue): expected_query = self.build_query(executor="CeleryExecutor", worker_queues=queue) assert query == expected_query - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"pollingInterval": 10}, "celery": {"keda": {"enabled": True}}}, - {"celery": {"keda": {"enabled": True, "pollingInterval": 10}}}, - {"keda": {"pollingInterval": 1}, "celery": {"keda": {"enabled": True, "pollingInterval": 10}}}, - ], - ) - def test_overwrite_keda_pooling_interval(self, workers_values): + def test_overwrite_keda_pooling_interval(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"keda": {"enabled": True, "pollingInterval": 10}}}}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.pollingInterval", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"cooldownPeriod": 10}, "celery": {"keda": {"enabled": True}}}, - {"celery": {"keda": {"enabled": True, "cooldownPeriod": 10}}}, - {"keda": {"cooldownPeriod": 1}, "celery": {"keda": {"enabled": True, "cooldownPeriod": 10}}}, - ], - ) - def test_overwrite_keda_cooldown_period(self, workers_values): + def test_overwrite_keda_cooldown_period(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"keda": {"enabled": True, "cooldownPeriod": 10}}}}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.cooldownPeriod", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"minReplicaCount": 10}, "celery": {"keda": {"enabled": True}}}, - {"celery": {"keda": {"enabled": True, "minReplicaCount": 10}}}, - {"keda": {"minReplicaCount": 1}, "celery": {"keda": {"enabled": True, "minReplicaCount": 10}}}, - ], - ) - def test_overwrite_keda_min_replica_count(self, workers_values): + def test_overwrite_keda_min_replica_count(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"keda": {"enabled": True, "minReplicaCount": 10}}}}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) assert jmespath.search("spec.minReplicaCount", docs[0]) == 10 - @pytest.mark.parametrize( - "workers_values", - [ - {"keda": {"maxReplicaCount": 5}, "celery": {"keda": {"enabled": True}}}, - {"celery": {"keda": {"enabled": True, "maxReplicaCount": 5}}}, - {"keda": {"maxReplicaCount": 1}, "celery": {"keda": {"enabled": True, "maxReplicaCount": 5}}}, - ], - ) - def test_overwrite_keda_max_replica_count(self, workers_values): + def test_overwrite_keda_max_replica_count(self): docs = render_chart( - values={"workers": workers_values}, + values={"workers": {"celery": {"keda": {"enabled": True, "maxReplicaCount": 5}}}}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) diff --git a/chart/tests/helm_tests/other/test_pgbouncer.py b/chart/tests/helm_tests/other/test_pgbouncer.py index 2219d2f0ca16c..da889e760c757 100644 --- a/chart/tests/helm_tests/other/test_pgbouncer.py +++ b/chart/tests/helm_tests/other/test_pgbouncer.py @@ -902,10 +902,10 @@ def test_should_create_pgbouncer_network_policy(self, executor): @pytest.mark.parametrize( "values", [ - {"workers": {"keda": {"enabled": True}}}, + {"workers": {"celery": {"keda": {"enabled": True}}}}, {"triggerer": {"keda": {"enabled": True}}}, { - "workers": {"keda": {"enabled": True}}, + "workers": {"celery": {"keda": {"enabled": True}}}, "triggerer": {"keda": {"enabled": True}}, }, ], @@ -936,38 +936,18 @@ def test_pod_selectors_with_keda_without_namespace_labels(self, executor, values "values", [ # test with workers.keda/workers.celery.keda enabled with namespace labels - { - "workers": { - "keda": {"namespaceLabels": {"app": "airflow"}}, - "celery": {"keda": {"enabled": True}}, - }, - }, { "workers": {"celery": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}}, }, - { - "workers": { - "keda": {"namespaceLabels": {"airflow": "app"}}, - "celery": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}, - }, - }, # test with triggerer.keda enabled with namespace labels {"triggerer": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}}, # test with workers.keda/workers.celery.keda and triggerer.keda both enabled with namespace labels - { - "workers": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}, - "triggerer": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}, - }, { "workers": {"celery": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}}, "triggerer": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}, }, # test with workers.keda/workers.celery.keda and triggerer.keda both enabled workers # with namespace labels and triggerer without namespace labels - { - "workers": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}, - "triggerer": {"keda": {"enabled": True}}, - }, { "workers": {"celery": {"keda": {"enabled": True, "namespaceLabels": {"app": "airflow"}}}}, "triggerer": {"keda": {"enabled": True}}, diff --git a/chart/tests/helm_tests/security/test_kerberos.py b/chart/tests/helm_tests/security/test_kerberos.py index 9978ae8659103..ad8bca571930c 100644 --- a/chart/tests/helm_tests/security/test_kerberos.py +++ b/chart/tests/helm_tests/security/test_kerberos.py @@ -19,7 +19,6 @@ import json import jmespath -import pytest from chart_utils.helm_template_generator import render_chart @@ -36,22 +35,13 @@ def test_kerberos_not_mentioned_in_render_if_disabled(self): k8s_objects_to_consider_str = json.dumps(k8s_objects_to_consider) assert k8s_objects_to_consider_str.count("kerberos") == 1 - @pytest.mark.parametrize( - "workers_values", - [ - {"kerberosSidecar": {"enabled": True}, "celery": {"persistence": {"enabled": True}}}, - {"celery": {"kerberosSidecar": {"enabled": True}, "persistence": {"enabled": True}}}, - { - "kerberosSidecar": {"enabled": True}, - "celery": {"kerberosSidecar": {"enabled": False}, "persistence": {"enabled": True}}, - }, - ], - ) - def test_kerberos_envs_available_in_worker_with_persistence(self, workers_values): + def test_kerberos_envs_available_in_worker_with_persistence(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": {"kerberosSidecar": {"enabled": True}, "persistence": {"enabled": True}} + }, "kerberos": { "enabled": True, "configPath": "/etc/krb5.conf", @@ -69,39 +59,18 @@ def test_kerberos_envs_available_in_worker_with_persistence(self, workers_values "spec.template.spec.containers[0].env", docs[0] ) - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosSidecar": { - "enabled": True, - "resources": {"requests": {"cpu": "200m", "memory": "200Mi"}}, - } - }, - { - "celery": { - "kerberosSidecar": { - "enabled": True, - "resources": {"requests": {"cpu": "200m", "memory": "200Mi"}}, - } - } - }, - { - "kerberosSidecar": {"resources": {"limits": {"cpu": "20m", "memory": "20Mi"}}}, - "celery": { - "kerberosSidecar": { - "enabled": True, - "resources": {"requests": {"cpu": "200m", "memory": "200Mi"}}, - } - }, - }, - ], - ) - def test_kerberos_sidecar_resources(self, workers_values): + def test_kerberos_sidecar_resources(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": workers_values, + "workers": { + "celery": { + "kerberosSidecar": { + "enabled": True, + "resources": {"requests": {"cpu": "200m", "memory": "200Mi"}}, + } + } + }, }, show_only=["templates/workers/worker-deployment.yaml"], ) diff --git a/chart/tests/helm_tests/security/test_rbac.py b/chart/tests/helm_tests/security/test_rbac.py index 4a4a7785b0921..26c9322a6ebbf 100644 --- a/chart/tests/helm_tests/security/test_rbac.py +++ b/chart/tests/helm_tests/security/test_rbac.py @@ -77,6 +77,7 @@ ("ServiceAccount", "test-rbac-cleanup"), ("ServiceAccount", "test-rbac-scheduler"), ("ServiceAccount", "test-rbac-worker"), + ("ServiceAccount", "test-rbac-worker-kubernetes"), ("ServiceAccount", "test-rbac-triggerer"), ("ServiceAccount", "test-rbac-pgbouncer"), ("ServiceAccount", "test-rbac-database-cleanup"), @@ -88,7 +89,7 @@ ("ServiceAccount", "test-rbac-redis"), ] -CUSTOM_SERVICE_ACCOUNT_NAMES = ( +CUSTOM_SERVICE_ACCOUNT_NAMES = [ (CUSTOM_SCHEDULER_NAME := "TestScheduler"), (CUSTOM_DAG_PROCESSOR_NAME := "TestDagProcessor"), (CUSTOM_API_SERVER_NAME := "TestAPISserver"), @@ -104,7 +105,9 @@ (CUSTOM_MIGRATE_DATABASE_JOBS_NAME := "TestMigrateDatabaseJob"), (CUSTOM_REDIS_NAME := "TestRedis"), (CUSTOM_POSTGRESQL_NAME := "TestPostgresql"), -) +] + +CUSTOM_WORKER_KUBERNETES_NAME = "TestWorkerKubernetes" class TestRBAC: @@ -118,6 +121,13 @@ def _get_object_tuples(self, sa: bool = True): return tuples + def _get_sa_tuples(self, sa: bool = True): + tuples = copy(CUSTOM_SERVICE_ACCOUNT_NAMES) + if sa: + tuples.append(CUSTOM_WORKER_KUBERNETES_NAME) + + return tuples + def test_deployments_no_rbac_no_sa(self): k8s_objects = render_chart( "test-rbac", @@ -147,7 +157,10 @@ def test_deployments_no_rbac_no_sa(self): "scheduler": {"serviceAccount": {"create": False}}, "dagProcessor": {"serviceAccount": {"create": False}}, "apiServer": {"serviceAccount": {"create": False}}, - "workers": {"serviceAccount": {"create": False}}, + "workers": { + "celery": {"serviceAccount": {"create": False}}, + "kubernetes": {"serviceAccount": {"create": False}}, + }, "triggerer": {"serviceAccount": {"create": False}}, "statsd": {"serviceAccount": {"create": False}}, "otelCollector": {"serviceAccount": {"create": False}}, @@ -201,7 +214,10 @@ def test_deployments_with_rbac_no_sa(self): "scheduler": {"serviceAccount": {"create": False}}, "dagProcessor": {"serviceAccount": {"create": False}}, "apiServer": {"serviceAccount": {"create": False}}, - "workers": {"serviceAccount": {"create": False}}, + "workers": { + "celery": {"serviceAccount": {"create": False}}, + "kubernetes": {"serviceAccount": {"create": False}}, + }, "triggerer": {"serviceAccount": {"create": False}}, "flower": {"enabled": True, "serviceAccount": {"create": False}}, "statsd": {"serviceAccount": {"create": False}}, @@ -284,7 +300,10 @@ def test_service_account_custom_names(self): "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "dagProcessor": {"serviceAccount": {"name": CUSTOM_DAG_PROCESSOR_NAME}}, "apiServer": {"serviceAccount": {"name": CUSTOM_API_SERVER_NAME}}, - "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, + "workers": { + "celery": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, + "kubernetes": {"serviceAccount": {"name": CUSTOM_WORKER_KUBERNETES_NAME}}, + }, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, "statsd": {"serviceAccount": {"name": CUSTOM_STATSD_NAME}}, @@ -306,7 +325,7 @@ def test_service_account_custom_names(self): for k8s_object in k8s_objects if k8s_object["kind"] == "ServiceAccount" ] - assert sorted(list_of_sa_names) == sorted(CUSTOM_SERVICE_ACCOUNT_NAMES) + assert sorted(list_of_sa_names) == sorted(self._get_sa_tuples()) def test_service_account_custom_names_in_objects(self): k8s_objects = render_chart( @@ -329,7 +348,7 @@ def test_service_account_custom_names_in_objects(self): "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "dagProcessor": {"serviceAccount": {"name": CUSTOM_DAG_PROCESSOR_NAME}}, "apiServer": {"serviceAccount": {"name": CUSTOM_API_SERVER_NAME}}, - "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, + "workers": {"celery": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, "statsd": {"serviceAccount": {"name": CUSTOM_STATSD_NAME}}, @@ -346,6 +365,7 @@ def test_service_account_custom_names_in_objects(self): "migrateDatabaseJob": {"serviceAccount": {"name": CUSTOM_MIGRATE_DATABASE_JOBS_NAME}}, }, ) + list_of_sa_names_in_objects = [] for k8s_object in k8s_objects: name = ( diff --git a/chart/tests/helm_tests/security/test_security_context.py b/chart/tests/helm_tests/security/test_security_context.py index 191c7fdb4d34a..c55a610eeab4a 100644 --- a/chart/tests/helm_tests/security/test_security_context.py +++ b/chart/tests/helm_tests/security/test_security_context.py @@ -148,7 +148,7 @@ def test_check_local_setting(self): "uid": 3000, "gid": 30, "securityContext": {"runAsUser": 6000, "fsGroup": 60}, - "workers": component_contexts, + "workers": {"celery": component_contexts}, "flower": {"enabled": True, **component_contexts}, "scheduler": component_contexts, "dagProcessor": component_contexts, @@ -177,22 +177,7 @@ def test_check_local_setting(self): # Test priority: # .securityContexts > securityContexts > uid + gid - @pytest.mark.parametrize( - "workers_values", - [ - {"securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}}, - {"celery": {"securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}}}, - { - "celery": { - "enableDefault": False, - "sets": [ - {"name": "test", "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} - ], - }, - }, - ], - ) - def test_check_local_setting_default_version(self, workers_values): + def test_check_local_setting_default_version(self): component_contexts = {"securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} docs = render_chart( values={ @@ -200,7 +185,7 @@ def test_check_local_setting_default_version(self, workers_values): "gid": 30, "securityContexts": {"pod": {"runAsUser": 6000, "fsGroup": 60}}, "apiServer": component_contexts, - "workers": workers_values, + "workers": {"celery": component_contexts}, "dagProcessor": component_contexts, "flower": {"enabled": True, **component_contexts}, "scheduler": component_contexts, @@ -304,25 +289,7 @@ def test_global_security_context(self): assert default_ctx_value_pod_redis == jmespath.search("spec.template.spec.securityContext", docs[-1]) # Test securityContexts for main containers - @pytest.mark.parametrize( - "workers_values", - [ - {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}, - {"celery": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, - { - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - ], - }, - }, - ], - ) - def test_main_container_setting(self, workers_values): + def test_main_container_setting(self): ctx_value = {"allowPrivilegeEscalation": False} security_context = {"securityContexts": {"container": ctx_value}} docs = render_chart( @@ -332,7 +299,7 @@ def test_main_container_setting(self, workers_values): "scheduler": security_context, "dagProcessor": security_context, "apiServer": security_context, - "workers": workers_values, + "workers": {"celery": security_context}, "flower": {"enabled": True, **security_context}, "statsd": security_context, "createUserJob": security_context, @@ -365,34 +332,13 @@ def test_main_container_setting(self, workers_values): assert ctx_value == jmespath.search("spec.template.spec.containers[0].securityContext", doc) # Test securityContexts for log-groomer-sidecar main container - @pytest.mark.parametrize( - "workers_values", - [ - {"logGroomerSidecar": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, - { - "celery": { - "logGroomerSidecar": { - "securityContexts": {"container": {"allowPrivilegeEscalation": False}} - } - } - }, - { - "logGroomerSidecar": {"securityContexts": {"container": {"runAsUser": 20}}}, - "celery": { - "logGroomerSidecar": { - "securityContexts": {"container": {"allowPrivilegeEscalation": False}} - } - }, - }, - ], - ) - def test_log_groomer_sidecar_container_setting(self, workers_values): + def test_log_groomer_sidecar_container_setting(self): ctx_value = {"allowPrivilegeEscalation": False} spec = {"logGroomerSidecar": {"securityContexts": {"container": ctx_value}}} docs = render_chart( values={ "scheduler": spec, - "workers": workers_values, + "workers": {"celery": spec}, "dagProcessor": spec, "triggerer": spec, }, @@ -423,37 +369,18 @@ def test_metrics_explorer_container_setting(self): assert ctx_value == jmespath.search("spec.template.spec.containers[1].securityContext", docs[0]) # Test securityContexts for worker-kerberos main container - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - }, - { - "celery": { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + def test_worker_kerberos_container_security_context(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "kerberosSidecar": { + "enabled": True, + "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + } } } }, - { - "kerberosSidecar": {"securityContexts": {"container": {"runAsUser": 10}}}, - "celery": { - "kerberosSidecar": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - }, - }, - ], - ) - def test_worker_kerberos_container_security_context(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -461,40 +388,18 @@ def test_worker_kerberos_container_security_context(self, workers_values): "allowPrivilegeEscalation": False } - @pytest.mark.parametrize( - "workers_values", - [ - { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, - } - }, - { - "celery": { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, + def test_worker_kerberos_init_container_security_contexts(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "kerberosInitContainer": { + "enabled": True, + "securityContexts": {"container": {"runAsUser": 2000}}, + } } } }, - { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - }, - "celery": { - "kerberosInitContainer": { - "enabled": True, - "securityContexts": {"container": {"runAsUser": 2000}}, - } - }, - }, - ], - ) - def test_worker_kerberos_init_container_security_contexts(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -502,28 +407,7 @@ def test_worker_kerberos_init_container_security_contexts(self, workers_values): "spec.template.spec.initContainers[?name=='kerberos-init'] | [0].securityContext", docs[0] ) == {"runAsUser": 2000} - @pytest.mark.parametrize( - "workers_values", - [ - {"waitForMigrations": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, - { - "celery": { - "waitForMigrations": { - "securityContexts": {"container": {"allowPrivilegeEscalation": False}} - } - } - }, - { - "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 0}}}, - "celery": { - "waitForMigrations": { - "securityContexts": {"container": {"allowPrivilegeEscalation": False}} - } - }, - }, - ], - ) - def test_wait_for_migrations_init_container_setting(self, workers_values): + def test_wait_for_migrations_init_container_setting(self): ctx_value = {"allowPrivilegeEscalation": False} spec = { "waitForMigrations": { @@ -537,7 +421,7 @@ def test_wait_for_migrations_init_container_setting(self, workers_values): "apiServer": spec, "triggerer": spec, "dagProcessor": spec, - "workers": workers_values, + "workers": {"celery": spec}, }, show_only=[ "templates/scheduler/scheduler-deployment.yaml", @@ -552,37 +436,19 @@ def test_wait_for_migrations_init_container_setting(self, workers_values): assert ctx_value == jmespath.search("spec.template.spec.initContainers[0].securityContext", doc) # Test securityContexts for volume-permissions init container - @pytest.mark.parametrize( - "workers_values", - [ - { - "persistence": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}, - "celery": {"persistence": {"enabled": True, "fixPermissions": True}}, - }, - { - "celery": { - "persistence": { - "enabled": True, - "fixPermissions": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + def test_volume_permissions_init_container_setting(self): + docs = render_chart( + values={ + "workers": { + "celery": { + "persistence": { + "enabled": True, + "fixPermissions": True, + "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + } } } }, - { - "persistence": {"securityContexts": {"container": {"allowPrivilegeEscalation": True}}}, - "celery": { - "persistence": { - "enabled": True, - "fixPermissions": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - }, - }, - ], - ) - def test_volume_permissions_init_container_setting(self, workers_values): - docs = render_chart( - values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -590,20 +456,7 @@ def test_volume_permissions_init_container_setting(self, workers_values): "allowPrivilegeEscalation": False } - @pytest.mark.parametrize( - "workers_values", - [ - {"securityContexts": {"pod": {"runAsUser": 7000}}}, - {"celery": {"securityContexts": {"pod": {"runAsUser": 7000}}}}, - { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "securityContexts": {"pod": {"runAsUser": 7000}}}], - }, - }, - ], - ) - def test_main_pod_setting(self, workers_values): + def test_main_pod_setting(self): ctx_value = {"runAsUser": 7000} security_context = {"securityContexts": {"pod": ctx_value}} docs = render_chart( @@ -612,7 +465,7 @@ def test_main_pod_setting(self, workers_values): "cleanup": {"enabled": True, **security_context}, "scheduler": security_context, "apiServer": security_context, - "workers": workers_values, + "workers": {"celery": security_context}, "flower": {"enabled": True, **security_context}, "statsd": security_context, "createUserJob": security_context, @@ -643,19 +496,7 @@ def test_main_pod_setting(self, workers_values): for doc in docs[1:]: assert ctx_value == jmespath.search("spec.template.spec.securityContext", doc) - @pytest.mark.parametrize( - "workers_values", - [ - {"securityContext": {"runAsUser": 7000}}, - { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "securityContext": {"runAsUser": 7000}}], - }, - }, - ], - ) - def test_main_pod_setting_legacy_security(self, workers_values): + def test_main_pod_setting_legacy_security(self): ctx_value = {"runAsUser": 7000} security_context = {"securityContext": ctx_value} docs = render_chart( @@ -663,7 +504,6 @@ def test_main_pod_setting_legacy_security(self, workers_values): "executor": "CeleryExecutor,KubernetesExecutor", "cleanup": {"enabled": True, **security_context}, "scheduler": security_context, - "workers": workers_values, "flower": {"enabled": True, **security_context}, "statsd": security_context, "createUserJob": security_context, @@ -676,7 +516,6 @@ def test_main_pod_setting_legacy_security(self, workers_values): "templates/cleanup/cleanup-cronjob.yaml", "templates/flower/flower-deployment.yaml", "templates/scheduler/scheduler-deployment.yaml", - "templates/workers/worker-deployment.yaml", "templates/statsd/statsd-deployment.yaml", "templates/jobs/create-user-job.yaml", "templates/jobs/migrate-database-job.yaml", @@ -713,29 +552,7 @@ def test_deprecated_overwrite_global(self): assert jmespath.search("spec.template.spec.securityContext.runAsUser", doc) == 9000 assert jmespath.search("spec.template.spec.securityContext.fsGroup", doc) == 90 - @pytest.mark.parametrize( - "workers_values", - [ - { - "securityContext": {"runAsUser": 6000, "fsGroup": 60}, - "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}, - }, - { - "securityContext": {"runAsUser": 6000, "fsGroup": 60}, - "celery": {"securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}}, - }, - { - "securityContext": {"runAsUser": 6000, "fsGroup": 60}, - "celery": { - "enableDefault": False, - "sets": [ - {"name": "test", "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} - ], - }, - }, - ], - ) - def test_deprecated_overwrite_local(self, workers_values): + def test_deprecated_overwrite_local(self): context = { "securityContext": {"runAsUser": 6000, "fsGroup": 60}, "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}, @@ -746,14 +563,12 @@ def test_deprecated_overwrite_local(self, workers_values): "flower": context, "scheduler": context, "triggerer": context, - "workers": workers_values, "dagProcessor": context, }, show_only=[ "templates/flower/flower-deployment.yaml", "templates/scheduler/scheduler-deployment.yaml", "templates/triggerer/triggerer-deployment.yaml", - "templates/workers/worker-deployment.yaml", "templates/dag-processor/dag-processor-deployment.yaml", ], ) @@ -762,13 +577,18 @@ def test_deprecated_overwrite_local(self, workers_values): assert jmespath.search("spec.template.spec.securityContext.runAsUser", doc) == 9000 assert jmespath.search("spec.template.spec.securityContext.fsGroup", doc) == 90 - def test_workers_overwrite_local(self): + def test_workers_sets_overwrite_local(self): docs = render_chart( values={ "workers": { - "securityContexts": {"pod": {"runAsUser": 6000, "fsGroup": 60}}, - "celery": {"securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}}, - }, + "celery": { + "enableDefault": False, + "securityContexts": {"pod": {"runAsUser": 6000, "fsGroup": 60}}, + "sets": [ + {"name": "test", "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} + ], + }, + } }, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -776,70 +596,22 @@ def test_workers_overwrite_local(self): assert jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0]) == 9000 assert jmespath.search("spec.template.spec.securityContext.fsGroup", docs[0]) == 90 - @pytest.mark.parametrize( - "values", - [ - { - "securityContexts": {"pod": {"runAsUser": 6000, "fsGroup": 60}}, - "celery": { - "enableDefault": False, - "sets": [ - {"name": "test", "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} - ], - }, - }, - { - "celery": { - "enableDefault": False, - "securityContexts": {"pod": {"runAsUser": 6000, "fsGroup": 60}}, - "sets": [ - {"name": "test", "securityContexts": {"pod": {"runAsUser": 9000, "fsGroup": 90}}} - ], - }, - }, - ], - ) - def test_workers_sets_overwrite_local(self, values): + def test_workers_sets_overwrite_local_container(self): docs = render_chart( - values={"workers": values}, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0]) == 9000 - assert jmespath.search("spec.template.spec.securityContext.fsGroup", docs[0]) == 90 - - @pytest.mark.parametrize( - "values", - [ - { - "securityContexts": {"container": {"allowPrivilegeEscalation": True}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "test", - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - ], - }, - }, - { - "celery": { - "enableDefault": False, - "securityContexts": {"container": {"allowPrivilegeEscalation": True}}, - "sets": [ - { - "name": "test", - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - } - ], - }, + values={ + "workers": { + "celery": { + "enableDefault": False, + "securityContexts": {"container": {"allowPrivilegeEscalation": True}}, + "sets": [ + { + "name": "test", + "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, + } + ], + }, + } }, - ], - ) - def test_workers_sets_overwrite_local_container(self, values): - docs = render_chart( - values={"workers": values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -864,21 +636,6 @@ def test_workers_sets_overwrite_local_container(self, values): ], } }, - { - "persistence": {"securityContexts": {"container": {"allowPrivilegeEscalation": True}}}, - "celery": { - "enableDefault": False, - "sets": [ - { - "name": "set1", - "persistence": { - "fixPermissions": True, - "securityContexts": {"container": {"allowPrivilegeEscalation": False}}, - }, - } - ], - }, - }, { "celery": { "enableDefault": False, diff --git a/chart/values.schema.json b/chart/values.schema.json index 4a645d85c22db..16f77588efa79 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1721,1086 +1721,6 @@ "x-docsSection": "Workers", "additionalProperties": false, "properties": { - "replicas": { - "description": "Number of Airflow Celery workers (deprecated, use `workers.celery.replicas` instead).", - "type": "integer", - "default": 1 - }, - "revisionHistoryLimit": { - "description": "Max number of old Airflow Celery workers ReplicaSets to retain (deprecated, use `workers.celery.revisionHistoryLimit` instead).", - "type": [ - "integer", - "null" - ], - "default": null, - "x-docsSection": "Kubernetes" - }, - "command": { - "description": "Command to use when running Airflow Celery workers and using pod-template-file (templated) (deprecated, use ``workers.celery.command`` and/or ``workers.kubernetes.command`` instead).", - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - }, - "default": null - }, - "args": { - "description": "Args to use when running Airflow Celery workers (templated) (deprecated, use `workers.celery.args` instead).", - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - }, - "default": [ - "bash", - "-c", - "exec \\\nairflow celery worker\n{{- if and .Values.workers.queue (ne .Values.workers.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.queue }}\n{{- end }}" - ] - }, - "livenessProbe": { - "description": "Liveness probe configuration for Airflow Celery worker containers (deprecated, use `workers.celery.livenessProbe` section instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Enable liveness probe for Airflow Celery workers (deprecated, use `workers.celery.livenessProbe.enabled` instead).", - "type": "boolean", - "default": true - }, - "initialDelaySeconds": { - "description": "Number of seconds after the container has started before liveness probes are initiated (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead).", - "type": "integer", - "default": 10 - }, - "timeoutSeconds": { - "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead).", - "type": "integer", - "default": 20 - }, - "failureThreshold": { - "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead).", - "type": "integer", - "default": 5 - }, - "periodSeconds": { - "description": "How often (in seconds) to perform the probe. Minimum value is 1 (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead).", - "type": "integer", - "default": 60 - }, - "command": { - "description": "Command for LivenessProbe (deprecated, use `workers.celery.livenessProbe.command` instead)", - "type": [ - "array", - "null" - ], - "default": null, - "items": { - "type": "string" - } - } - } - }, - "updateStrategy": { - "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a StatefulSet (deprecated, use `workers.celery.updateStrategy` instead).", - "type": [ - "null", - "object" - ], - "default": null - }, - "podManagementPolicy": { - "description": "Specifies the policy for managing pods within the Airflow Celery worker (deprecated, use `workers.celery.podManagementPolicy` instead). Only applicable to StatefulSet.", - "type": [ - "null", - "string" - ], - "default": null, - "enum": [ - "OrderedReady", - "Parallel" - ] - }, - "strategy": { - "description": "Specifies the strategy used to replace old Airflow Celery worker pods by new ones when deployed as a Deployment (deprecated, use `workers.celery.strategy` instead).", - "type": [ - "null", - "object" - ], - "default": { - "rollingUpdate": { - "maxSurge": "100%", - "maxUnavailable": "50%" - } - } - }, - "serviceAccount": { - "description": "Create ServiceAccount for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.serviceAccount`` and/or ``workers.kubernetes.serviceAccount`` instead).", - "type": "object", - "properties": { - "automountServiceAccountToken": { - "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods (deprecated, use ``workers.celery.serviceAccount.automountServiceAccountToken`` and/or ``workers.kubernetes.serviceAccount.automountServiceAccountToken`` instead)", - "type": "boolean", - "default": true - }, - "create": { - "description": "Specifies whether a ServiceAccount should be created (deprecated, use ``workers.celery.serviceAccount.create`` and/or ``workers.kubernetes.serviceAccount.create`` instead).", - "type": "boolean", - "default": true - }, - "name": { - "description": "The name of the ServiceAccount to use (deprecated, use ``workers.celery.serviceAccount.name`` and/or ``workers.kubernetes.serviceAccount.name`` instead). If not set and create is true, a name is generated using the release name.", - "type": [ - "string", - "null" - ], - "default": null - }, - "annotations": { - "description": "Annotations to add to the worker Kubernetes ServiceAccount (deprecated, use ``workers.celery.serviceAccount.annotations`` and/or ``workers.kubernetes.serviceAccount.annotations`` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - } - } - }, - "keda": { - "description": "KEDA configuration of Airflow Celery workers (deprecated, use `workers.celery.keda` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Allow KEDA autoscaling (deprecated, use `workers.celery.keda.enabled` instead).", - "type": "boolean", - "default": false - }, - "namespaceLabels": { - "description": "Labels used in `matchLabels` for namespace in the PgBouncer NetworkPolicy (deprecated, use `workers.celery.keda.namespaceLabels` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "pollingInterval": { - "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA (deprecated, use `workers.celery.keda.pollingInterval` instead).", - "type": "integer", - "default": 5 - }, - "cooldownPeriod": { - "description": "How many seconds KEDA will wait before scaling to zero (deprecated, use `workers.celery.keda.cooldownPeriod` instead).", - "type": "integer", - "default": 30 - }, - "minReplicaCount": { - "description": "Minimum number of Airflow Celery workers created by KEDA (deprecated, use `workers.celery.keda.minReplicaCount` instead).", - "type": "integer", - "default": 0 - }, - "maxReplicaCount": { - "description": "Maximum number of Airflow Celery workers created by KEDA (deprecated, use `workers.celery.keda.maxReplicaCount` instead).", - "type": "integer", - "default": 10 - }, - "advanced": { - "description": "Advanced KEDA configuration (deprecated, use `workers.celery.keda.advanced` instead).", - "type": "object", - "default": {}, - "additionalProperties": false, - "properties": { - "horizontalPodAutoscalerConfig": { - "description": "HorizontalPodAutoscalerConfig specifies horizontal scale config (deprecated, use `workers.celery.keda.advanced.horizontalPodAutoscalerConfig` instead).", - "type": "object", - "default": {}, - "properties": { - "behavior": { - "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use `workers.celery.keda.advanced.horizontalPodAutoscalerConfig.behavior` instead).", - "type": "object", - "default": {}, - "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" - } - } - } - } - }, - "query": { - "description": "Query to use for KEDA autoscaling (deprecated, use `workers.celery.keda.query` instead). Must return a single integer.", - "type": "string", - "default": "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') AND queue IN ( {{- range $i, $q := splitList \",\" .Values.workers.queue -}} {{- if $i }},{{ end }}'{{ $q | trim }}' {{- end -}} ) {{- if contains \"KubernetesExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'KubernetesExecutor' {{- else if contains \"airflow.providers.edge3.executors.EdgeExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'EdgeExecutor' {{- end }}" - }, - "usePgbouncer": { - "description": "Weather to use PGBouncer to connect to the database or not when it is enabled (deprecated, use `workers.celery.keda.usePgbouncer` instead). This configuration will be ignored if PGBouncer is not enabled.", - "type": "boolean", - "default": true - } - } - }, - "hpa": { - "description": "HPA configuration for Airflow Celery workers (deprecated, use ``workers.celery.hpa`` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Allow HPA autoscaling (KEDA must be disabled) (deprecated, use ``workers.celery.hpa.enabled`` instead).", - "type": "boolean", - "default": false - }, - "minReplicaCount": { - "description": "Minimum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.minReplicaCount`` instead).", - "type": "integer", - "default": 0 - }, - "maxReplicaCount": { - "description": "Maximum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.maxReplicaCount`` instead).", - "type": "integer", - "default": 5 - }, - "metrics": { - "description": "Specifications for which to use to calculate the desired replica count (deprecated, use ``workers.celery.hpa.metrics`` instead).", - "type": "array", - "default": [ - { - "type": "Resource", - "resource": { - "name": "cpu", - "target": { - "type": "Utilization", - "averageUtilization": 80 - } - } - } - ], - "items": { - "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" - } - }, - "behavior": { - "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use ``workers.celery.hpa.behavior`` instead).", - "type": "object", - "default": {}, - "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" - } - } - }, - "persistence": { - "description": "Persistence configuration for Airflow Celery workers (deprecated, use `workers.celery.persistence` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Enable persistent volumes (deprecated, use `workers.celery.persistence.enabled` instead).", - "type": "boolean", - "default": true - }, - "persistentVolumeClaimRetentionPolicy": { - "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", - "description": "PersistentVolumeClaim retention policy to be used in the lifecycle of a StatefulSet (deprecated, use `workers.celery.persistence.persistentVolumeClaimRetentionPolicy` instead)." - }, - "size": { - "description": "Volume size for Airflow Celery worker StatefulSet (deprecated, use `workers.celery.persistence.size` instead).", - "type": "string", - "default": "100Gi" - }, - "storageClassName": { - "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated) (deprecated, use `workers.celery.persistence.storageClassName` instead).", - "type": [ - "string", - "null" - ], - "default": null - }, - "fixPermissions": { - "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner (deprecated, use `workers.celery.persistence.fixPermissions` instead).", - "type": "boolean", - "default": false - }, - "annotations": { - "description": "Annotations to add to Airflow Celery worker volumes (deprecated, use `workers.celery.persistence.annotations` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "securityContexts": { - "description": "Security context definition for the persistence. If not set, the values from global `securityContexts` will be used (deprecated, use `workers.celery.persistence.securityContexts` instead).", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "container": { - "description": "Container security context definition for the persistence.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - } - } - }, - "kerberosSidecar": { - "description": "Kerberos sidecar for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.kerberosSidecar`` and/or ``workers.kubernetes.kerberosSidecar`` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Enable Kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.enabled`` and/or ``workers.kubernetes.kerberosSidecar.enabled`` instead).", - "type": "boolean", - "default": false - }, - "resources": { - "description": "Resources on kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.resources`` and/or ``workers.kubernetes.kerberosSidecar.resources`` instead).", - "type": "object", - "default": {}, - "examples": [ - { - "limits": { - "cpu": "100m", - "memory": "128Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - } - ], - "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" - }, - "containerLifecycleHooks": { - "description": "Container Lifecycle Hooks definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.containerLifecycleHooks`` and/or ``workers.kubernetes.kerberosSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "postStart": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo postStart handler > /usr/share/message" - ] - } - }, - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo preStop handler > /usr/share/message" - ] - } - } - } - ] - }, - "securityContexts": { - "description": "Security context definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.securityContexts`` and/or ``workers.kubernetes.kerberosSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "container": { - "description": "Container security context definition for the kerberos sidecar (deprecated, use ``workers.celery.kerberosSidecar.securityContexts.container`` and/or ``workers.kubernetes.kerberosSidecar.securityContexts.container`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - } - } - }, - "kerberosInitContainer": { - "description": "Kerberos init container for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.kerberosInitContainer`` and/or ``workers.kubernetes.kerberosInitContainer`` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Enable Kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.enabled`` and/or ``workers.kubernetes.kerberosInitContainer.enabled`` instead).", - "type": "boolean", - "default": false - }, - "resources": { - "description": "Resources on kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.resources`` and/or ``workers.kubernetes.kerberosInitContainer.resources`` instead).", - "type": "object", - "default": {}, - "examples": [ - { - "limits": { - "cpu": "100m", - "memory": "128Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - } - ], - "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" - }, - "containerLifecycleHooks": { - "description": "Container Lifecycle Hooks definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.containerLifecycleHooks`` and/or ``workers.kubernetes.kerberosInitContainer.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "postStart": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo postStart handler > /usr/share/message" - ] - } - }, - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo preStop handler > /usr/share/message" - ] - } - } - } - ] - }, - "securityContexts": { - "description": "Security context definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.securityContexts`` and/or ``workers.kubernetes.kerberosInitContainer.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "container": { - "description": "Container security context definition for the kerberos init container (deprecated, use ``workers.celery.kerberosInitContainer.securityContexts.container`` and/or ``workers.kubernetes.kerberosInitContainer.securityContexts.container`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - } - } - }, - "podDisruptionBudget": { - "description": "Worker pod disruption budget (deprecated, use `workers.celery.podDisruptionBudget` instead).", - "type": "object", - "additionalProperties": true, - "properties": { - "enabled": { - "description": "Enable pod disruption budget (deprecated, use `workers.celery.podDisruptionBudget.enabled` instead).", - "type": "boolean", - "default": false - }, - "config": { - "description": "Disruption budget configuration (deprecated, use `workers.celery.podDisruptionBudget.config` instead).", - "type": "object", - "additionalProperties": true, - "properties": { - "maxUnavailable": { - "description": "Max unavailable pods for worker (deprecated, use `workers.celery.podDisruptionBudget.config.maxUnavailable` instead).", - "type": [ - "integer", - "string" - ], - "default": 1 - }, - "minAvailable": { - "description": "Min available pods for worker (deprecated, use `workers.celery.podDisruptionBudget.config.minAvailable` instead).", - "type": [ - "integer", - "string" - ], - "default": 1 - } - } - } - } - }, - "resources": { - "description": "Resource configuration for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.resources`` or/and ``workers.kubernetes.resources`` instead).", - "type": "object", - "default": {}, - "examples": [ - { - "limits": { - "cpu": "100m", - "memory": "128Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - } - ], - "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" - }, - "terminationGracePeriodSeconds": { - "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes. It is used by Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.terminationGracePeriodSeconds`` or/and ``workers.kubernetes.terminationGracePeriodSeconds`` instead).", - "type": "integer", - "default": 600 - }, - "safeToEvict": { - "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down. It is used by Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.safeToEvict`` or/and ``workers.kubernetes.safeToEvict`` instead).", - "type": "boolean", - "default": false - }, - "extraContainers": { - "description": "Launch additional containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraContainers`` and/or ``workers.kubernetes.extraContainers`` instead). Note, if used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.Container" - } - }, - "extraInitContainers": { - "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraInitContainers`` and/or ``workers.kubernetes.extraInitContainers`` instead).", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.Container" - } - }, - "extraPorts": { - "description": "Expose additional ports of Airflow Celery worker container (deprecated, use `workers.celery.extraPorts` instead).", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" - } - }, - "extraVolumes": { - "description": "Additional volumes attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumes`` and/or ``workers.kubernetes.extraVolumes`` instead).", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.Volume" - } - }, - "extraVolumeMounts": { - "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumeMounts`` and/or ``workers.kubernetes.extraVolumeMounts`` instead).", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" - } - }, - "nodeSelector": { - "description": "Select certain nodes for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.nodeSelector`` or/and ``workers.kubernetes.nodeSelector`` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "runtimeClassName": { - "description": "Specify runtime for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.runtimeClassName`` and/or ``workers.kubernetes.runtimeClassName`` instead).", - "type": [ - "string", - "null" - ], - "default": null - }, - "priorityClassName": { - "description": "Specify priority for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.priorityClassName`` and/or ``workers.kubernetes.priorityClassName`` instead).", - "type": [ - "string", - "null" - ], - "default": null - }, - "affinity": { - "description": "Specify scheduling constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.affinity`` and/or ``workers.kubernetes.affinity`` instead).", - "type": "object", - "default": "See values.yaml", - "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" - }, - "tolerations": { - "description": "Specify Tolerations for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.tolerations`` and/or ``workers.kubernetes.tolerations`` instead).", - "type": "array", - "default": [], - "items": { - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" - } - }, - "topologySpreadConstraints": { - "description": "Specify topology spread constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.topologySpreadConstraints`` and/or ``workers.kubernetes.topologySpreadConstraints`` instead).", - "type": "array", - "default": [], - "items": { - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" - } - }, - "hostAliases": { - "description": "Specify HostAliases for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.hostAliases`` and/or ``workers.kubernetes.hostAliases`` instead).", - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" - }, - "type": "array", - "default": [], - "examples": [ - { - "ip": "127.0.0.2", - "hostnames": [ - "test.hostname.one" - ] - }, - { - "ip": "127.0.0.3", - "hostnames": [ - "test.hostname.two" - ] - } - ] - }, - "annotations": { - "description": "Annotations to add to the Airflow Celery worker deployment (deprecated, use ``workers.celery.annotations`` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "podAnnotations": { - "description": "Annotations to add to the Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.podAnnotations`` and/or ``workers.kubernetes.podAnnotations`` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "schedulerName": { - "description": "Specify kube scheduler name for Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.schedulerName`` and/or ``workers.kubernetes.schedulerName`` instead).", - "type": [ - "string", - "null" - ], - "default": null, - "x-docsSection": "Common" - }, - "labels": { - "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.labels`` and/or ``workers.kubernetes.labels`` instead).", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "logGroomerSidecar": { - "description": "Configuration for Airflow Celery worker log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar`` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Whether to deploy the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.enabled`` instead).", - "type": "boolean", - "default": true - }, - "command": { - "description": "Command to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.command`` instead).", - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - }, - "default": null - }, - "args": { - "description": "Args to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.args`` instead).", - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - }, - "default": [ - "bash", - "/clean-logs" - ] - }, - "retentionDays": { - "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", - "type": "integer", - "default": 15, - "minimum": 0 - }, - "retentionMinutes": { - "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", - "type": "integer", - "default": 0, - "minimum": 0 - }, - "frequencyMinutes": { - "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.frequencyMinutes`` instead).", - "type": "integer", - "default": 15 - }, - "maxSizeBytes": { - "description": "Max size of logs directory in bytes (deprecated, use ``workers.celery.logGroomerSidecar.maxSizeBytes`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", - "type": "integer", - "default": 0, - "minimum": 0 - }, - "maxSizePercent": { - "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", - "type": "integer", - "default": 0, - "minimum": 0, - "maximum": 100 - }, - "env": { - "description": "Add additional env vars to log groomer sidecar container (templated) (deprecated, use ``workers.celery.logGroomerSidecar.env`` instead).", - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" - }, - "type": "array", - "default": [], - "x-kubernetes-patch-merge-key": "name", - "x-kubernetes-patch-strategy": "merge" - }, - "resources": { - "description": "Resources for Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.resources`` instead).", - "type": "object", - "default": {}, - "examples": [ - { - "limits": { - "cpu": "100m", - "memory": "128Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - } - ], - "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" - }, - "containerLifecycleHooks": { - "description": "Container Lifecycle Hooks definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "postStart": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo postStart handler > /usr/share/message" - ] - } - }, - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo preStop handler > /usr/share/message" - ] - } - } - } - ] - }, - "securityContexts": { - "description": "Security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "container": { - "description": "Container security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts.container`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - } - } - }, - "securityContext": { - "description": "Security context for the Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from `securityContext` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", - "default": {}, - "examples": [ - { - "runAsUser": 50000, - "runAsGroup": 0, - "fsGroup": 0 - } - ] - }, - "containerLifecycleHooks": { - "description": "Container Lifecycle Hooks definition for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.containerLifecycleHooks`` and/or ``workers.kubernetes.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "postStart": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo postStart handler > /usr/share/message" - ] - } - }, - "preStop": { - "exec": { - "command": [ - "/bin/sh", - "-c", - "echo preStop handler > /usr/share/message" - ] - } - } - } - ] - }, - "securityContexts": { - "description": "Security context definition for the Airflow Celery workers and pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "pod": { - "description": "Pod security context definition (deprecated, use ``workers.celery.securityContexts.pod`` and/or ``workers.kubernetes.securityContexts.pod`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "runAsUser": 50000, - "runAsGroup": 0, - "fsGroup": 0 - } - ] - }, - "container": { - "description": "Container security context definition (deprecated, use ``workers.celery.securityContexts.container`` and/or ``workers.kubernetes.securityContexts.container`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - }, - "waitForMigrations": { - "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers (deprecated, use ``workers.celery.waitForMigrations`` instead).", - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "description": "Enable wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.enabled`` instead).", - "type": "boolean", - "default": true - }, - "env": { - "description": "Add additional env vars to wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.env`` instead).", - "type": "array", - "default": [], - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "required": [ - "name", - "value" - ], - "additionalProperties": false - } - }, - "securityContexts": { - "description": "Security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", - "type": "object", - "x-docsSection": "Kubernetes", - "properties": { - "container": { - "description": "Container security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts.container`` instead).", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", - "default": {}, - "x-docsSection": "Kubernetes", - "examples": [ - { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - } - } - ] - } - } - } - } - }, - "env": { - "description": "Add additional env vars to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.env`` and/or ``workers.kubernetes.env`` instead).", - "type": "array", - "default": [], - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - }, - "valueFrom": { - "type": "object", - "properties": { - "configMapKeyRef": { - "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", - "description": "Selects a key of a ConfigMap." - }, - "secretKeyRef": { - "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", - "description": "Selects a key of a secret in the pod's namespace" - } - }, - "anyOf": [ - { - "required": [ - "configMapKeyRef" - ] - }, - { - "required": [ - "secretKeyRef" - ] - } - ] - } - }, - "required": [ - "name" - ], - "anyOf": [ - { - "required": [ - "value" - ] - }, - { - "required": [ - "valueFrom" - ] - } - ], - "additionalProperties": false - } - }, - "volumeClaimTemplates": { - "description": "Specify additional volume claim template for Airflow Celery workers (deprecated, use ``workers.celery.volumeClaimTemplates`` instead).", - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate" - }, - "examples": [ - { - "name": "data-volume-1", - "storageClassName": "storage-class-1", - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "10Gi" - } - } - }, - { - "name": "data-volume-2", - "storageClassName": "storage-class-2", - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "20Gi" - } - } - } - ] - }, "celery": { "description": "Airflow Celery Workers configuration.", "type": "object", @@ -2808,11 +1728,8 @@ "properties": { "replicas": { "description": "Number of Airflow Celery workers.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 1 }, "revisionHistoryLimit": { "description": "Max number of old Airflow Celery workers ReplicaSets to retain.", @@ -2843,7 +1760,11 @@ "items": { "type": "string" }, - "default": null + "default": [ + "bash", + "-c", + "exec \\\nairflow celery worker\n{{- if and .Values.workers.celery.queue (ne .Values.workers.celery.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.celery.queue }}\n{{- end }}" + ] }, "livenessProbe": { "description": "Liveness probe configuration for Airflow Celery worker containers.", @@ -2852,43 +1773,28 @@ "properties": { "enabled": { "description": "Enable liveness probe for Airflow Celery workers.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "initialDelaySeconds": { - "description": "Number of seconds after the container has started before liveness probes are initiated.", - "type": [ - "integer", - "null" - ], - "default": null + "description": "Number of seconds after the container has started before liveness probes are initiated.", + "type": "integer", + "default": 10 }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 20 }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 5 }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Minimum value is 1.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 60 }, "command": { "description": "Command for LivenessProbe", @@ -2917,7 +1823,12 @@ "null", "object" ], - "default": null + "default": { + "rollingUpdate": { + "maxSurge": "100%", + "maxUnavailable": "50%" + } + } }, "podManagementPolicy": { "description": "Specifies the policy for managing pods within the Airflow Celery worker. Only applicable to StatefulSet.", @@ -3022,11 +1933,8 @@ "properties": { "enabled": { "description": "Enable pod disruption budget.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "config": { "description": "Disruption budget configuration.", @@ -3037,19 +1945,17 @@ "description": "Max unavailable pods for worker.", "type": [ "integer", - "string", - "null" + "string" ], - "default": null + "default": 1 }, "minAvailable": { "description": "Min available pods for worker.", "type": [ "integer", - "string", - "null" + "string" ], - "default": null + "default": 1 } } } @@ -3061,19 +1967,13 @@ "properties": { "automountServiceAccountToken": { "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "create": { "description": "Specifies whether a ServiceAccount should be created.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "name": { "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", @@ -3100,11 +2000,8 @@ "properties": { "enabled": { "description": "Allow KEDA autoscaling.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "namespaceLabels": { "description": "Labels used in `matchLabels` for namespace in the PgBouncer NetworkPolicy.", @@ -3116,35 +2013,23 @@ }, "pollingInterval": { "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 5 }, "cooldownPeriod": { "description": "How many seconds KEDA will wait before scaling to zero.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 30 }, "minReplicaCount": { "description": "Minimum number of Airflow Celery workers created by KEDA.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 0 }, "maxReplicaCount": { "description": "Maximum number of Airflow Celery workers created by KEDA.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 10 }, "advanced": { "description": "Advanced KEDA configuration.", @@ -3169,19 +2054,13 @@ }, "query": { "description": "Query to use for KEDA autoscaling. Must return a single integer.", - "type": [ - "string", - "null" - ], - "default": null + "type": "string", + "default": "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') AND queue IN ( {{- range $i, $q := splitList \",\" .Values.workers.celery.queue -}} {{- if $i }},{{ end }}'{{ $q | trim }}' {{- end -}} ) {{- if contains \"KubernetesExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'KubernetesExecutor' {{- else if contains \"airflow.providers.edge3.executors.EdgeExecutor\" .Values.executor }} AND executor IS DISTINCT FROM 'EdgeExecutor' {{- end }}" }, "usePgbouncer": { "description": "Weather to use PGBouncer to connect to the database or not when it is enabled. This configuration will be ignored if PGBouncer is not enabled.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true } } }, @@ -3192,35 +2071,34 @@ "properties": { "enabled": { "description": "Allow HPA autoscaling (KEDA must be disabled).", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "minReplicaCount": { "description": "Minimum number of Airflow Celery workers created by HPA.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 0 }, "maxReplicaCount": { "description": "Maximum number of Airflow Celery workers created by HPA.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 5 }, "metrics": { "description": "Specifications for which to use to calculate the desired replica count.", - "type": [ - "array", - "null" + "type": "array", + "default": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 80 + } + } + } ], - "default": null, "items": { "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" } @@ -3240,11 +2118,8 @@ "properties": { "enabled": { "description": "Enable persistent volumes.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "persistentVolumeClaimRetentionPolicy": { "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", @@ -3252,11 +2127,8 @@ }, "size": { "description": "Volume size for Airflow Celery worker StatefulSet.", - "type": [ - "string", - "null" - ], - "default": null + "type": "string", + "default": "100Gi" }, "storageClassName": { "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated).", @@ -3268,11 +2140,8 @@ }, "fixPermissions": { "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "annotations": { "description": "Annotations to add to Airflow Celery worker volumes.", @@ -3315,11 +2184,8 @@ "properties": { "enabled": { "description": "Enable Kerberos sidecar.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "resources": { "description": "Resources on kerberos sidecar.", @@ -3401,11 +2267,8 @@ "properties": { "enabled": { "description": "Enable Kerberos init container.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "resources": { "description": "Resources on kerberos init container.", @@ -3500,19 +2363,13 @@ }, "terminationGracePeriodSeconds": { "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 600 }, "safeToEvict": { "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "extraContainers": { "description": "Launch additional containers into Airflow Celery worker (templated).", @@ -3655,11 +2512,8 @@ "properties": { "enabled": { "description": "Whether to deploy the Airflow log groomer sidecar.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "command": { "description": "Command to use when running the Airflow log groomer sidecar (templated).", @@ -3681,50 +2535,38 @@ "items": { "type": "string" }, - "default": [] + "default": [ + "bash", + "/clean-logs" + ] }, "retentionDays": { "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", - "type": [ - "integer", - "null" - ], - "default": null, + "type": "integer", + "default": 15, "minimum": 0 }, "retentionMinutes": { "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", - "type": [ - "integer", - "null" - ], - "default": null, + "type": "integer", + "default": 0, "minimum": 0 }, "frequencyMinutes": { "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 15 }, "maxSizeBytes": { "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", - "type": [ - "integer", - "null" - ], - "default": null, + "type": "integer", + "default": 0, "minimum": 0 }, "maxSizePercent": { "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", - "type": [ - "integer", - "null" - ], - "default": null, + "type": "integer", + "default": 0, "minimum": 0, "maximum": 100 }, @@ -3818,11 +2660,8 @@ "properties": { "enabled": { "description": "Whether to create init container to wait for db migrations.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "env": { "description": "Add additional env vars to wait-for-airflow-migrations init container.", @@ -4060,19 +2899,13 @@ "properties": { "automountServiceAccountToken": { "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods. If not specified, the ``workers.serviceAccount.automountServiceAccountToken`` value will be taken.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "create": { "description": "Specifies whether a ServiceAccount should be created. If not specified, the ServiceAccount will be generated and used from ``templates/workers/worker-serviceaccount.yaml`` file if ``workers.serviceAccount.create`` will be 'true'.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": true }, "name": { "description": "The name of the ServiceAccount to use. If not set and ``create`` is 'true', a name is generated using the release name with kubernetes dedicated name.", @@ -4099,11 +2932,8 @@ "properties": { "enabled": { "description": "Enable Kerberos sidecar.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "resources": { "description": "Resources on kerberos sidecar.", @@ -4185,11 +3015,8 @@ "properties": { "enabled": { "description": "Enable kerberos init container.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "resources": { "description": "Resources on kerberos init container.", @@ -4284,19 +3111,13 @@ }, "terminationGracePeriodSeconds": { "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", - "type": [ - "integer", - "null" - ], - "default": null + "type": "integer", + "default": 600 }, "safeToEvict": { "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", - "type": [ - "boolean", - "null" - ], - "default": null + "type": "boolean", + "default": false }, "extraContainers": { "description": "Launch additional containers into pods created with pod-template-file (templated). Note, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", diff --git a/chart/values.yaml b/chart/values.yaml index 0df514c93ece7..6305c8e9546f7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -632,587 +632,9 @@ kerberos: # Airflow Worker Config workers: - # Number of Airflow Celery workers (deprecated, use `workers.celery.replicas` instead) - replicas: 1 - - # Max number of old Airflow Celery workers ReplicaSets to retain - # (deprecated, use `workers.celery.revisionHistoryLimit` instead) - revisionHistoryLimit: ~ - - # Command to use when running Airflow Celery workers and using pod-template-file (templated) - # (deprecated, use `workers.celery.command` and/or `workers.kubernetes.command` instead) - command: ~ - - # Args to use when running Airflow Celery workers (templated) - # (deprecated, use `workers.celery.args` instead) - args: - - "bash" - - "-c" - # The format below is necessary to get `helm lint` happy - - |- - exec \ - airflow celery worker - {{- if and .Values.workers.queue (ne .Values.workers.queue "default") }} - {{- " -q " }}{{ .Values.workers.queue }} - {{- end }} - - # If the Airflow Celery worker stops responding for 5 minutes (5*60s) - # kill the worker and let Kubernetes restart it - # (deprecated, use `workers.celery.livenessProbe` section instead) - livenessProbe: - # (deprecated, use `workers.celery.livenessProbe.enabled` instead) - enabled: true - # (deprecated, use `workers.celery.livenessProbe.initialDelaySeconds` instead) - initialDelaySeconds: 10 - # (deprecated, use `workers.celery.livenessProbe.timeoutSeconds` instead) - timeoutSeconds: 20 - # (deprecated, use `workers.celery.livenessProbe.failureThreshold` instead) - failureThreshold: 5 - # (deprecated, use `workers.celery.livenessProbe.periodSeconds` instead) - periodSeconds: 60 - # (deprecated, use `workers.celery.livenessProbe.command` instead) - command: ~ - - # Update Strategy when Airflow Celery worker is deployed as a StatefulSet - # (deprecated, use `workers.celery.updateStrategy` instead) - updateStrategy: ~ - # Update Strategy when Airflow Celery worker is deployed as a Deployment - # (deprecated, use `workers.celery.strategy` instead) - strategy: - rollingUpdate: - maxSurge: "100%" - maxUnavailable: "50%" - - # Allow relaxing ordering guarantees for Airflow Celery worker while preserving its uniqueness and identity - # (deprecated, use `workers.celery.podManagementPolicy` instead) - # podManagementPolicy: Parallel - - # When not set, the values defined in the global securityContext will - # be used in Airflow Celery workers and pod-template-file - # (deprecated, use `workers.celery.securityContexts` and/or `workers.kubernetes.securityContexts` instead) - securityContext: {} - # runAsUser: 50000 - # fsGroup: 0 - # runAsGroup: 0 - - # Detailed default security context for the - # Airflow Celery workers and pod-template-file on container and pod level - # (deprecated, use `workers.celery.securityContexts` and/or `workers.kubernetes.securityContexts` instead) - securityContexts: - # (deprecated, use - # `workers.celery.securityContexts.pod` and/or - # `workers.kubernetes.securityContexts.pod` - # instead) - pod: {} - # (deprecated, use - # `workers.celery.securityContexts.container` and/or - # `workers.kubernetes.securityContexts.container` - # instead) - container: {} - - # Container level Lifecycle Hooks definition for - # Airflow Celery workers and pods created with pod-template-file - # (deprecated, use - # `workers.celery.containerLifecycleHooks` and/or - # `workers.kubernetes.containerLifecycleHooks` - # instead) - containerLifecycleHooks: {} - - # Airflow Celery workers pod disruption budget - # (deprecated, use `workers.celery.podDisruptionBudget` instead) - podDisruptionBudget: - # (deprecated, use `workers.celery.podDisruptionBudget.enabled` instead) - enabled: false - - # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) - # (deprecated, use `workers.celery.podDisruptionBudget.config` instead) - config: - # (deprecated, use `workers.celery.podDisruptionBudget.config.maxUnavailable` instead) - maxUnavailable: 1 - - # (deprecated, use `workers.celery.podDisruptionBudget.config.minAvailable` instead) - # minAvailable: 1 - - # Create Service Account for Airflow Celery workers and pods created with pod-template-file - # (deprecated, use `workers.celery.serviceAccount` and/or `workers.kubernetes.serviceAccount` instead) - serviceAccount: - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - # (deprecated, use - # `workers.celery.serviceAccount.automountServiceAccountToken` and/or - # `workers.kubernetes.serviceAccount.automountServiceAccountToken` - # instead) - automountServiceAccountToken: true - - # Specifies whether a Service Account should be created - # (deprecated, use - # `workers.celery.serviceAccount.create` and/or - # `workers.kubernetes.serviceAccount.create` - # instead) - create: true - - # The name of the Service Account to use. - # If not set and `create` is 'true', a name is generated using the release name - # (deprecated, use - # `workers.celery.serviceAccount.name` and/or - # `workers.kubernetes.serviceAccount.name` - # instead) - name: ~ - - # Annotations to add to worker Kubernetes Service Account. - # (deprecated, use - # `workers.celery.serviceAccount.annotations` and/or - # `workers.kubernetes.serviceAccount.annotations` - # instead) - annotations: {} - - # Allow KEDA autoscaling for Airflow Celery workers - # (deprecated, use `workers.celery.keda` instead) - keda: - # (deprecated, use `workers.celery.keda.enabled` instead) - enabled: false - - # (deprecated, use `workers.celery.keda.namespaceLabels` instead) - namespaceLabels: {} - - # How often KEDA polls the Airflow DB to report new scale requests to the HPA - # (deprecated, use `workers.celery.keda.pollingInterval` instead) - pollingInterval: 5 - - # How many seconds KEDA will wait before scaling to zero. - # Note: HPA has a separate cooldown period for scale-downs - # (deprecated, use `workers.celery.keda.cooldownPeriod` instead) - cooldownPeriod: 30 - - # Minimum number of Airflow Celery workers created by keda - # (deprecated, use `workers.celery.keda.minReplicaCount` instead) - minReplicaCount: 0 - - # Maximum number of Airflow Celery workers created by keda - # (deprecated, use `workers.celery.keda.maxReplicaCount` instead) - maxReplicaCount: 10 - - # Specify HPA related options - # (deprecated, use `workers.celery.keda.advanced` instead) - advanced: {} - # horizontalPodAutoscalerConfig: - # behavior: - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Percent - # value: 100 - # periodSeconds: 15 - - # Query to use for KEDA autoscaling. Must return a single integer. - # (deprecated, use `workers.celery.keda.query` instead) - query: >- - SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) - FROM task_instance - WHERE (state='running' OR state='queued') - AND queue IN ( - {{- range $i, $q := splitList "," .Values.workers.queue -}} - {{- if $i }},{{ end }}'{{ $q | trim }}' - {{- end -}} - ) - {{- if contains "KubernetesExecutor" .Values.executor }} - AND executor IS DISTINCT FROM 'KubernetesExecutor' - {{- else if contains "airflow.providers.edge3.executors.EdgeExecutor" .Values.executor }} - AND executor IS DISTINCT FROM 'EdgeExecutor' - {{- end }} - - # Weather to use PGBouncer to connect to the database or not when it is enabled - # This configuration will be ignored if PGBouncer is not enabled - # (deprecated, use `workers.celery.keda.usePgbouncer` instead) - usePgbouncer: true - - # Allow HPA for Airflow Celery workers (KEDA must be disabled) - # (deprecated, use `workers.celery.hpa` instead) - hpa: - # (deprecated, use `workers.celery.hpa.enabled` instead) - enabled: false - - # Minimum number of Airflow Celery workers created by HPA - # (deprecated, use `workers.celery.hpa.minReplicaCount` instead) - minReplicaCount: 0 - - # Maximum number of Airflow Celery workers created by HPA - # (deprecated, use `workers.celery.hpa.maxReplicaCount` instead) - maxReplicaCount: 5 - - # Specifications for which to use to calculate the desired replica count - # (deprecated, use `workers.celery.hpa.metrics` instead) - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 - - # Scaling behavior of the target in both Up and Down directions - # (deprecated, use `workers.celery.hpa.behavior` instead) - behavior: {} - - # Persistence volume configuration for Airflow Celery workers - # (deprecated, use `workers.celery.persistence` instead) - persistence: - # Enable persistent volumes (deprecated, use `workers.celery.persistence.enabled` instead) - enabled: true - - # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed - # (deprecated, use `workers.celery.persistence.persistentVolumeClaimRetentionPolicy` instead) - persistentVolumeClaimRetentionPolicy: ~ - # persistentVolumeClaimRetentionPolicy: - # whenDeleted: Delete - # whenScaled: Delete - - # Volume size for Airflow Celery worker StatefulSet - # (deprecated, use `workers.celery.persistence.size` instead) - size: 100Gi - - # If using a custom storageClass, pass name ref to all StatefulSets here - # (deprecated, use `workers.celery.persistence.storageClassName` instead) - storageClassName: - - # Execute init container to chown log directory. - # This is currently only needed in kind, due to usage - # of local-path provisioner. - # (deprecated, use `workers.celery.persistence.fixPermissions` instead) - fixPermissions: false - - # Annotations to add to Airflow Celery worker volumes - # (deprecated, use `workers.celery.persistence.annotations` instead) - annotations: {} - - # Detailed default security context for persistence on container level - # (deprecated, use `workers.celery.persistence.securityContexts` instead) - securityContexts: - # (deprecated, use `workers.celery.persistence.securityContexts.container` instead) - container: {} - - # Kerberos sidecar configuration for Airflow Celery workers and pods created with pod-template-file - # (deprecated, use `workers.celery.kerberosSidecar` and/or `workers.kubernetes.kerberosSidecar` instead) - kerberosSidecar: - # Enable kerberos sidecar - # (deprecated, use - # `workers.celery.kerberosSidecar.enabled` and/or - # `workers.kubernetes.kerberosSidecar.enabled` - # instead) - enabled: false - - # (deprecated, use - # `workers.celery.kerberosSidecar.resources` and/or - # `workers.kubernetes.kerberosSidecar.resources` - # instead) - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - # Detailed default security context for kerberos sidecar on container level - # (deprecated, use - # `workers.celery.kerberosSidecar.securityContexts` and/or - # `workers.kubernetes.kerberosSidecar.securityContexts` - # instead) - securityContexts: - # (deprecated, use - # `workers.celery.kerberosSidecar.securityContexts.container` and/or - # `workers.kubernetes.kerberosSidecar.securityContexts.container` - # instead) - container: {} - - # Container level lifecycle hooks - # (deprecated, use - # `workers.celery.kerberosSidecar.containerLifecycleHooks` and/or - # `workers.kubernetes.kerberosSidecar.containerLifecycleHooks` - # instead) - containerLifecycleHooks: {} - - # Kerberos init container configuration for Airflow Celery workers and pods created with pod-template-file - # (deprecated, use - # `workers.celery.kerberosInitContainer` and/or - # `workers.kubernetes.kerberosInitContainer` - # instead) - kerberosInitContainer: - # Enable kerberos init container - # (deprecated, use - # `workers.celery.kerberosInitContainer.enabled` and/or - # `workers.kubernetes.kerberosInitContainer.enabled` - # instead) - enabled: false - - # (deprecated, use - # `workers.celery.kerberosInitContainer.resources` and/or - # `workers.kubernetes.kerberosInitContainer.resources` - # instead) - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - # Detailed default security context for kerberos init container - # (deprecated, use - # `workers.celery.kerberosInitContainer.securityContexts` and/or - # `workers.kubernetes.kerberosInitContainer.securityContexts` - # instead) - securityContexts: - # (deprecated, use - # `workers.celery.kerberosInitContainer.securityContexts.container` and/or - # `workers.kubernetes.kerberosInitContainer.securityContexts.container` - # instead) - container: {} - - # Container level lifecycle hooks - # (deprecated, use - # `workers.celery.kerberosInitContainer.containerLifecycleHooks` and/or - # `workers.kubernetes.kerberosInitContainer.containerLifecycleHooks` - # instead) - containerLifecycleHooks: {} - - # Resource configuration for Airflow Celery workers and pods created with pod-template-file - # (deprecated, use `workers.celery.resources` and/or `workers.kubernetes.resources` instead) - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - # Grace period for tasks to finish after SIGTERM is sent from Kubernetes. - # It is used by Airflow Celery workers and pod-template-file. - # (deprecated, use - # `workers.celery.terminationGracePeriodSeconds` and/or - # `workers.kubernetes.terminationGracePeriodSeconds` - # instead) - terminationGracePeriodSeconds: 600 - - # This setting tells Kubernetes that its ok to evict when it wants to scale a node down. - # It is used by Airflow Celery workers and pod-template-file. - # (deprecated, use - # `workers.celery.safeToEvict` and/or - # `workers.kubernetes.safeToEvict` - # instead) - safeToEvict: false - - # Launch additional containers into Airflow Celery worker - # and pods created with pod-template-file (templated). - # (deprecated, use - # `workers.celery.extraContainers` and/or - # `workers.kubernetes.extraContainers` - # instead) - # Note: If used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main - # container finishes so Airflow can continue the worker shutdown process! - extraContainers: [] - - # Add additional init containers into Airflow Celery workers - # and pods created with pod-template-file (templated). - # (deprecated, use - # `workers.celery.extraInitContainers` and/or - # `workers.kubernetes.extraInitContainers` - # instead) - extraInitContainers: [] - - # Additional volumes attached to the Airflow Celery workers - # and pods created with pod-template-file - # (deprecated, use `workers.celery.extraVolumes` and/or `workers.kubernetes.extraVolumes` instead) - extraVolumes: [] - # Mount additional volumes into workers pods. It can be templated like in the following example: - # extraVolumes: - # - name: my-templated-extra-volume - # secret: - # secretName: '{{ include "my_secret_template" . }}' - # defaultMode: 0640 - # optional: true - - # Additional volume mounts attached to the Airflow Celery workers - # and pods created with pod-template-file - # (deprecated, use - # `workers.celery.extraVolumeMounts` and/or - # `workers.kubernetes.extraVolumeMounts` - # instead) - extraVolumeMounts: [] - # Mount additional volumes into workers pods. It can be templated like in the following example: - # extraVolumeMounts: - # - name: my-templated-extra-volume - # mountPath: "{{ .Values.my_custom_path }}" - # readOnly: true - - # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. - # (deprecated, use `workers.celery.extraPorts` instead) - extraPorts: [] - - # Select certain nodes for Airflow Celery worker pods and pods created with pod-template-file - # (deprecated, use `workers.celery.nodeSelector` and/or `workers.kubernetes.nodeSelector` instead) - nodeSelector: {} - - # (deprecated, use `workers.celery.runtimeClassName` and/or `workers.kubernetes.runtimeClassName` instead) - runtimeClassName: ~ - - # (deprecated, use `workers.celery.priorityClassName` and/or `workers.kubernetes.priorityClassName` instead) - priorityClassName: ~ - - # (deprecated, use `workers.celery.affinity` and/or `workers.kubernetes.affinity` instead) - affinity: {} - # Default Airflow Celery worker affinity is: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - podAffinityTerm: - # labelSelector: - # matchLabels: - # component: worker - # topologyKey: kubernetes.io/hostname - # weight: 100 - - # (deprecated, use `workers.celery.tolerations` and/or `workers.kubernetes.tolerations` instead) - tolerations: [] - - # (deprecated, use - # `workers.celery.topologySpreadConstraints` and/or - # `workers.kubernetes.topologySpreadConstraints` - # instead) - topologySpreadConstraints: [] - - # hostAliases to use in Airflow Celery worker pods and pods created with pod-template-file - # (deprecated, use `workers.celery.hostAliases` and/or `workers.kubernetes.hostAliases` instead) - # See: - # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ - hostAliases: [] - # - ip: "127.0.0.2" - # hostnames: - # - "test.hostname.one" - # - ip: "127.0.0.3" - # hostnames: - # - "test.hostname.two" - - # Annotations for the Airflow Celery worker resource - # (deprecated, use `workers.celery.annotations` instead) - annotations: {} - - # Pod annotations for the Airflow Celery workers and pods created with pod-template-file (templated) - # (deprecated, use `workers.celery.podAnnotations` and/or `workers.kubernetes.podAnnotations` instead) - podAnnotations: {} - - # Labels specific to Airflow Celery workers objects and pods created with pod-template-file - # (deprecated, use `workers.celery.labels` and/or `workers.kubernetes.labels` instead) - labels: {} - - # Log groomer configuration for Airflow Celery workers - # (deprecated, use `workers.celery.logGroomerSidecar` instead) - logGroomerSidecar: - # Whether to deploy the Airflow Celery worker log groomer sidecar - # (deprecated, use `workers.celery.logGroomerSidecar.enabled` instead) - enabled: true - - # Command to use when running the Airflow Celery worker log groomer sidecar (templated) - # (deprecated, use `workers.celery.logGroomerSidecar.command` instead) - command: ~ - - # Args to use when running the Airflow Celery worker log groomer sidecar (templated) - # (deprecated, use `workers.celery.logGroomerSidecar.args` instead) - args: ["bash", "/clean-logs"] - - # Number of days to retain logs - # (deprecated, use `workers.celery.logGroomerSidecar.retentionDays` instead) - retentionDays: 15 - - # Number of minutes to retain logs. - # This can be used for finer granularity than days. - # Total retention is `retentionDays` + `retentionMinutes`. - # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes` instead) - retentionMinutes: 0 - - # Frequency to attempt to groom logs (in minutes) - # (deprecated, use `workers.celery.logGroomerSidecar.frequencyMinutes` instead) - frequencyMinutes: 15 - - # Max size of logs in bytes. 0 = disabled - # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead) - maxSizeBytes: 0 - - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. - # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent` instead) - maxSizePercent: 0 - - # (deprecated, use `workers.celery.logGroomerSidecar.resources` instead) - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - # Detailed default security context for `logGroomerSidecar` for container level - # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts` instead) - securityContexts: - # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts.container` instead) - container: {} - - # (deprecated, use `workers.celery.logGroomerSidecar.env` instead) - env: [] - - # Container level lifecycle hooks - # (deprecated, use `workers.celery.logGroomerSidecar.containerLifecycleHooks` instead) - containerLifecycleHooks: {} - - # Configuration of wait-for-airflow-migration init container for Airflow Celery workers - # (deprecated, use `workers.celery.waitForMigrations` instead) - waitForMigrations: - # Whether to create init container to wait for db migrations - # (deprecated, use `workers.celery.waitForMigrations.enabled` instead) - enabled: true - - # (deprecated, use `workers.celery.waitForMigrations.env` instead) - env: [] - - # Detailed default security context for wait-for-airflow-migrations container - # (deprecated, use `workers.celery.waitForMigrations.securityContexts` instead) - securityContexts: - # (deprecated, use `workers.celery.waitForMigrations.securityContexts.container` instead) - container: {} - - # Additional env variable configuration for Airflow Celery workers and pods created with pod-template-file - # (deprecated, use `workers.celery.env` and/or `workers.kubernetes.env` instead) - env: [] - - # Additional volume claim templates for Airflow Celery workers. - # Requires mounting of specified volumes under extraVolumeMounts. - # (deprecated, use `workers.celery.volumeClaimTemplates` instead) - volumeClaimTemplates: [] - # Volume Claim Templates example: - # volumeClaimTemplates: - # - metadata: - # name: data-volume-1 - # spec: - # storageClassName: "storage-class-1" - # accessModes: - # - "ReadWriteOnce" - # resources: - # requests: - # storage: "10Gi" - # - metadata: - # name: data-volume-2 - # spec: - # storageClassName: "storage-class-2" - # accessModes: - # - "ReadWriteOnce" - # resources: - # requests: - # storage: "20Gi" - - # (deprecated, use `workers.celery.schedulerName` and/or `workers.kubernetes.schedulerName` instead) - schedulerName: ~ - celery: # Number of Airflow Celery workers - replicas: ~ + replicas: 1 # Max number of old Airflow Celery workers ReplicaSets to retain revisionHistoryLimit: ~ @@ -1221,16 +643,25 @@ workers: command: ~ # Args to use when running Airflow Celery workers (templated) - args: ~ + args: + - "bash" + - "-c" + # The format below is necessary to get `helm lint` happy + - |- + exec \ + airflow celery worker + {{- if and .Values.workers.celery.queue (ne .Values.workers.celery.queue "default") }} + {{- " -q " }}{{ .Values.workers.celery.queue }} + {{- end }} # If the Airflow Celery worker stops responding for 5 minutes (5*60s) # kill the worker and let Kubernetes restart it livenessProbe: - enabled: ~ - initialDelaySeconds: ~ - timeoutSeconds: ~ - failureThreshold: ~ - periodSeconds: ~ + enabled: true + initialDelaySeconds: 10 + timeoutSeconds: 20 + failureThreshold: 5 + periodSeconds: 60 command: ~ # Enable the default workers defined by the root `workers` and `workers.celery` @@ -1269,7 +700,10 @@ workers: # Update Strategy when Airflow Celery worker is deployed as a StatefulSet updateStrategy: ~ # Update Strategy when Airflow Celery worker is deployed as a Deployment - strategy: ~ + strategy: + rollingUpdate: + maxSurge: "100%" + maxUnavailable: "50%" # Allow relaxing ordering guarantees for Airflow Celery worker # while preserving its uniqueness and identity @@ -1286,20 +720,20 @@ workers: # Airflow Celery workers pod disruption budget podDisruptionBudget: - enabled: ~ + enabled: false # PDB configuration (`minAvailable` and `maxUnavailable` are mutually exclusive) config: - maxUnavailable: ~ + maxUnavailable: 1 # minAvailable: ~ # Create Service Account for Airflow Celery workers serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - automountServiceAccountToken: ~ + automountServiceAccountToken: true # Specifies whether a Service Account should be created - create: ~ + create: true # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name @@ -1310,22 +744,22 @@ workers: # Allow KEDA autoscaling for Airflow Celery workers keda: - enabled: ~ + enabled: false namespaceLabels: {} # How often KEDA polls the airflow DB to report new scale requests to the HPA - pollingInterval: ~ + pollingInterval: 5 # How many seconds KEDA will wait before scaling to zero. # Note: HPA has a separate cooldown period for scale-downs - cooldownPeriod: ~ + cooldownPeriod: 30 # Minimum number of Airflow Celery workers created by KEDA - minReplicaCount: ~ + minReplicaCount: 0 # Maximum number of Airflow Celery workers created by KEDA - maxReplicaCount: ~ + maxReplicaCount: 10 # Specify HPA related options advanced: {} @@ -1339,24 +773,43 @@ workers: # periodSeconds: 15 # Query to use for KEDA autoscaling. Must return a single integer - query: ~ + query: >- + SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) + FROM task_instance + WHERE (state='running' OR state='queued') + AND queue IN ( + {{- range $i, $q := splitList "," .Values.workers.celery.queue -}} + {{- if $i }},{{ end }}'{{ $q | trim }}' + {{- end -}} + ) + {{- if contains "KubernetesExecutor" .Values.executor }} + AND executor IS DISTINCT FROM 'KubernetesExecutor' + {{- else if contains "airflow.providers.edge3.executors.EdgeExecutor" .Values.executor }} + AND executor IS DISTINCT FROM 'EdgeExecutor' + {{- end }} # Weather to use PGBouncer to connect to the database or not when it is enabled # This configuration will be ignored if PGBouncer is not enabled - usePgbouncer: ~ + usePgbouncer: true # Allow HPA for Airflow Celery workers (KEDA must be disabled) hpa: - enabled: ~ + enabled: false # Minimum number of Airflow Celery workers created by HPA - minReplicaCount: ~ + minReplicaCount: 0 # Maximum number of Airflow Celery workers created by HPA - maxReplicaCount: ~ + maxReplicaCount: 5 # Specifications for which to use to calculate the desired replica count - metrics: ~ + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 # Scaling behavior of the target in both Up and Down directions behavior: {} @@ -1364,7 +817,7 @@ workers: # Persistence volume configuration for Airflow Celery workers persistence: # Enable persistent volumes - enabled: ~ + enabled: true # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed persistentVolumeClaimRetentionPolicy: ~ @@ -1373,15 +826,15 @@ workers: # whenScaled: Delete # Volume size for Airflow Celery worker StatefulSet - size: ~ + size: 100Gi # If using a custom storageClass, pass name ref to all StatefulSets here - storageClassName: + storageClassName: ~ # Execute init container to chown log directory. # This is currently only needed in kind, due to usage # of local-path provisioner. - fixPermissions: ~ + fixPermissions: false # Annotations to add to Airflow Celery worker volumes annotations: {} @@ -1393,7 +846,7 @@ workers: # Kerberos sidecar configuration for Airflow Celery workers kerberosSidecar: # Enable kerberos sidecar - enabled: ~ + enabled: false resources: {} # limits: @@ -1415,7 +868,7 @@ workers: kerberosInitContainer: # Enable kerberos init container # If `workers.kerberosInitContainer.enabled` is set to True, this flag has no effect - enabled: ~ + enabled: false resources: {} # limits: @@ -1442,10 +895,10 @@ workers: # memory: 128Mi # Grace period for tasks to finish after SIGTERM is sent from Kubernetes - terminationGracePeriodSeconds: ~ + terminationGracePeriodSeconds: 600 # This setting tells Kubernetes that its ok to evict when it wants to scale a node down - safeToEvict: ~ + safeToEvict: false # Launch additional containers into Airflow Celery worker (templated) extraContainers: [] @@ -1519,30 +972,30 @@ workers: # Log groomer configuration for Airflow Celery workers logGroomerSidecar: # Whether to deploy the Airflow Celery worker log groomer sidecar - enabled: ~ + enabled: true # Command to use when running the Airflow Celery worker log groomer sidecar (templated) command: ~ # Args to use when running the Airflow Celery worker log groomer sidecar (templated) - args: [] + args: ["bash", "/clean-logs"] # Number of days to retain logs - retentionDays: ~ + retentionDays: 15 # Number of minutes to retain logs. # This can be used for finer granularity than days. # Total retention is `retentionDays` + `retentionMinutes`. - retentionMinutes: ~ + retentionMinutes: 0 # Frequency to attempt to groom logs (in minutes) - frequencyMinutes: ~ + frequencyMinutes: 15 # Max size of logs in bytes. 0 = disabled - maxSizeBytes: ~ + maxSizeBytes: 0 # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. - maxSizePercent: ~ + maxSizePercent: 0 resources: {} # limits: @@ -1564,7 +1017,7 @@ workers: # Configuration of wait-for-airflow-migration init container for Airflow Celery workers waitForMigrations: # Whether to create init container to wait for db migrations - enabled: ~ + enabled: true env: [] @@ -1620,13 +1073,13 @@ workers: serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ # If not specified, the `workers.serviceAccount.automountServiceAccountToken` value will be taken - automountServiceAccountToken: ~ + automountServiceAccountToken: true # Specifies whether a Service Account should be created. # If not specified, the Service Account will be generated and used from # 'templates/workers/worker-serviceaccount.yaml' file if `workers.serviceAccount.create` # will be 'true' - create: ~ + create: true # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name @@ -1640,7 +1093,7 @@ workers: # Kerberos sidecar configuration for pods created with pod-template-file kerberosSidecar: # Enable kerberos sidecar - enabled: ~ + enabled: false resources: {} # limits: @@ -1662,7 +1115,7 @@ workers: kerberosInitContainer: # Enable kerberos init container # If `workers.kerberosInitContainer.enabled` is set to True, this flag has no effect - enabled: ~ + enabled: false resources: {} # limits: @@ -1689,10 +1142,10 @@ workers: # memory: 128Mi # Grace period for tasks to finish after SIGTERM is sent from Kubernetes - terminationGracePeriodSeconds: ~ + terminationGracePeriodSeconds: 600 # This setting tells Kubernetes that its ok to evict when it wants to scale a node down - safeToEvict: ~ + safeToEvict: false # Launch additional containers into pods created with pod-template-file (templated). # Note: You are responsible for signaling sidecars to exit when the main