From 5f9b59c5b381507f7536a3189acf207ee8a6d575 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:22:13 -0700 Subject: [PATCH 01/21] Add rpc server to helm chart --- chart/templates/_helpers.yaml | 10 + .../rpc-server/rpc-server-deployment.yaml | 259 ++++++ .../rpc-server/rpc-server-ingress.yaml | 113 +++ .../rpc-server/rpc-server-networkpolicy.yaml | 59 ++ .../rpc-server-poddisruptionbudget.yaml | 46 ++ .../rpc-server/rpc-server-service.yaml | 59 ++ .../rpc-server/rpc-server-serviceaccount.yaml | 41 + chart/values.schema.json | 744 ++++++++++++++++++ chart/values.yaml | 123 +++ 9 files changed, 1454 insertions(+) create mode 100644 chart/templates/rpc-server/rpc-server-deployment.yaml create mode 100644 chart/templates/rpc-server/rpc-server-ingress.yaml create mode 100644 chart/templates/rpc-server/rpc-server-networkpolicy.yaml create mode 100644 chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml create mode 100644 chart/templates/rpc-server/rpc-server-service.yaml create mode 100644 chart/templates/rpc-server/rpc-server-serviceaccount.yaml diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 59a28fd9b7df4..f834fbc5a1309 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -582,6 +582,16 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- end }} + +{{/* Create the name of the RPC server service account to use */}} +{{- define "rpcServer.serviceAccountName" -}} + {{- if .Values.rpcServer.serviceAccount.create }} + {{- default (printf "%s-rpc-server" (include "airflow.serviceAccountName" .)) .Values.rpcServer.serviceAccount.name }} + {{- else }} + {{- default "default" .Values.rpcServer.serviceAccount.name }} + {{- end }} +{{- end }} + {{/* Create the name of the redis service account to use */}} {{- define "redis.serviceAccountName" -}} {{- if .Values.redis.serviceAccount.create }} diff --git a/chart/templates/rpc-server/rpc-server-deployment.yaml b/chart/templates/rpc-server/rpc-server-deployment.yaml new file mode 100644 index 0000000000000..78813ce91228c --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-deployment.yaml @@ -0,0 +1,259 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow rpc-server Deployment +################################# +{{- if .Values.rpcServer.enabled }} +{{- $nodeSelector := or .Values.rpcServer.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.rpcServer.affinity .Values.affinity }} +{{- $tolerations := or .Values.rpcServer.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.rpcServer.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := or .Values.rpcServer.revisionHistoryLimit .Values.revisionHistoryLimit }} +{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.rpcServer) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.rpcServer) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.rpcServer.waitForMigrations) }} +{{- $containerLifecycleHooks := or .Values.rpcServer.containerLifecycleHooks .Values.containerLifecycleHooks }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-rpc-server + labels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.rpcServer.annotations }} + annotations: {{- toYaml .Values.rpcServer.annotations | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.rpcServer.replicas }} + {{- if $revisionHistoryLimit }} + revisionHistoryLimit: {{ $revisionHistoryLimit }} + {{- end }} + strategy: + {{- if .Values.rpcServer.strategy }} + {{- toYaml .Values.rpcServer.strategy | nindent 4 }} + {{- else }} + {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} + # Here we define the rolling update strategy + # - maxSurge define how many pod we can add at a time + # - maxUnavailable define how many pod can be unavailable + # during the rolling update + # Setting maxUnavailable to 0 would make sure we have the appropriate + # capacity during the rolling update. + # You can also use percentage based value instead of integer. + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + {{- else }} + type: Recreate + {{- end }} + {{- end }} + selector: + matchLabels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 8 }} + {{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }} + checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }} + checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }} + {{- if .Values.airflowPodAnnotations }} + {{- toYaml .Values.airflowPodAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.rpcServer.podAnnotations }} + {{- toYaml .Values.rpcServer.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rpcServer.hostAliases }} + hostAliases: {{- toYaml .Values.rpcServer.hostAliases | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "rpcServer.serviceAccountName" . }} + {{- if .Values.rpcServer.priorityClassName }} + priorityClassName: {{ .Values.rpcServer.priorityClassName }} + {{- end }} + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} + affinity: + {{- if $affinity }} + {{- toYaml $affinity | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + component: rpc-server + topologyKey: kubernetes.io/hostname + weight: 100 + {{- end }} + tolerations: {{- toYaml $tolerations | nindent 8 }} + topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} + restartPolicy: Always + securityContext: {{ $securityContext | nindent 8 }} + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + initContainers: + {{- if .Values.rpcServer.waitForMigrations.enabled }} + - name: wait-for-airflow-migrations + resources: {{- toYaml .Values.rpcServer.resources | nindent 12 }} + image: {{ template "airflow_image_for_migrations" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.rpcServer.extraVolumeMounts }} + {{- tpl (toYaml .Values.rpcServer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + args: {{- include "wait-for-migrations-command" . | indent 10 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.rpcServer.waitForMigrations.env }} + {{- tpl (toYaml .Values.rpcServer.waitForMigrations.env) $ | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.rpcServer.extraInitContainers }} + {{- toYaml .Values.rpcServer.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: rpc-server + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + securityContext: {{ $containerSecurityContext | nindent 12 }} + {{- if $containerLifecycleHooks }} + lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} + {{- end }} + {{- if .Values.rpcServer.command }} + command: {{ tpl (toYaml .Values.rpcServer.command) . | nindent 12 }} + {{- end }} + {{- if .Values.rpcServer.args }} + args: {{- tpl (toYaml .Values.rpcServer.args) . | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.rpcServer.resources | nindent 12 }} + volumeMounts: + {{- include "airflow_config_mount" . | nindent 12 }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.rpcServer.extraVolumeMounts }} + {{- tpl (toYaml .Values.rpcServer.extraVolumeMounts) . | nindent 12 }} + {{- end }} + ports: + - name: rpc-server + containerPort: {{ .Values.ports.rpcServer }} + livenessProbe: + httpGet: + path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports.rpcServer }} + {{- if .Values.rpcServer.baseUrl}} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + {{- end }} + scheme: {{ .Values.rpcServer.livenessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.rpcServer.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.rpcServer.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.rpcServer.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.rpcServer.livenessProbe.periodSeconds }} + readinessProbe: + httpGet: + path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports.rpcServer }} + {{- if .Values.rpcServer.baseUrl }} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + {{- end }} + scheme: {{ .Values.rpcServer.readinessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values.rpcServer.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.rpcServer.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.rpcServer.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.rpcServer.readinessProbe.periodSeconds }} + startupProbe: + httpGet: + path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports.rpcServer }} + {{- if .Values.rpcServer.baseUrl}} + httpHeaders: + - name: Host + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + {{- end }} + scheme: {{ .Values.rpcServer.startupProbe.scheme | default "http" }} + timeoutSeconds: {{ .Values.rpcServer.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values.rpcServer.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.rpcServer.startupProbe.periodSeconds }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + {{- include "container_extra_envs" (list . .Values.rpcServer.env) | indent 10 }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }} + {{- include "git_sync_container" . | nindent 8 }} + {{- end }} + {{- if .Values.rpcServer.extraContainers }} + {{- toYaml .Values.rpcServer.extraContainers | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} + {{- if (semverCompare "<2.0.0" .Values.airflowVersion) }} + {{- end }} + {{- if .Values.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ template "airflow_logs_volume_claim" . }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- if .Values.rpcServer.extraVolumes }} + {{- tpl (toYaml .Values.rpcServer.extraVolumes) . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-ingress.yaml b/chart/templates/rpc-server/rpc-server-ingress.yaml new file mode 100644 index 0000000000000..17c2c70297a07 --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-ingress.yaml @@ -0,0 +1,113 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow rpc-server Ingress +################################# +{{- if .Values.rpcServer.enabled }} +{{- if or .Values.ingress.rpcServer.enabled .Values.ingress.enabled }} +{{- $fullname := (include "airflow.fullname" .) }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }}-ingress + labels: + tier: airflow + component: airflow-ingress + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.ingress.rpcServer.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.rpcServer.hosts (.Values.ingress.rpcServer.hosts | first | kindIs "string" | not) }} + {{- $anyTlsHosts := false -}} + {{- range .Values.ingress.rpcServer.hosts }} + {{- if .tls }} + {{- if .tls.enabled }} + {{- $anyTlsHosts = true -}} + {{- end }} + {{- end }} + {{- end }} + {{- if $anyTlsHosts }} + tls: + {{- range .Values.ingress.rpcServer.hosts }} + {{- if .tls }} + {{- if .tls.enabled }} + - hosts: + - {{ .name | quote }} + secretName: {{ .tls.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- else if .Values.ingress.rpcServer.tls.enabled }} + tls: + - hosts: + {{- .Values.ingress.rpcServer.hosts | default (list .Values.ingress.rpcServer.host) | toYaml | nindent 8 }} + secretName: {{ .Values.ingress.rpcServer.tls.secretName }} + {{- end }} + rules: + {{- range .Values.ingress.rpcServer.hosts | default (list .Values.ingress.rpcServer.host) }} + - http: + paths: + {{- range $.Values.ingress.rpcServer.precedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ .serviceName }} + port: + name: {{ .servicePort }} + {{- end }} + - backend: + service: + name: {{ $fullname }}-rpc-server + port: + name: airflow-ui + {{- if $.Values.ingress.rpcServer.path }} + path: {{ $.Values.ingress.rpcServer.path }} + pathType: {{ $.Values.ingress.rpcServer.pathType }} + {{- end }} + {{- range $.Values.ingress.rpcServer.succeedingPaths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ .serviceName }} + port: + name: {{ .servicePort }} + {{- end }} + {{- $hostname := . -}} + {{- if . | kindIs "string" | not }} + {{- $hostname = .name -}} + {{- end }} + {{- if $hostname }} + host: {{ tpl $hostname $ | quote }} + {{- end }} + {{- end }} + {{- if .Values.ingress.rpcServer.ingressClassName }} + ingressClassName: {{ .Values.ingress.rpcServer.ingressClassName }} + {{- end }} +{{- end }} +{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-networkpolicy.yaml b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml new file mode 100644 index 0000000000000..1bcde16b16b87 --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow rpc-server NetworkPolicy +################################# +{{- if .Values.rpcServer.enabled }} +{{- if .Values.networkPolicies.enabled }} +{{- $from := or .Values.rpcServer.networkPolicy.ingress.from .Values.rpcServer.extraNetworkPolicies }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "airflow.fullname" . }}-rpc-server-policy + labels: + tier: airflow + component: airflow-rpc-server-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if $from }} + ingress: + - from: {{- toYaml $from | nindent 6 }} + ports: + {{ range .Values.rpcServer.networkPolicy.ingress.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml b/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml new file mode 100644 index 0000000000000..bf722b89542fd --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml @@ -0,0 +1,46 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow rpc-server PodDisruptionBudget +################################# +{{- if .Values.rpcServer.enabled }} +{{- if .Values.rpcServer.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "airflow.fullname" . }}-rpc-server-pdb + labels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + {{- toYaml .Values.rpcServer.podDisruptionBudget.config | nindent 2 }} +{{- end }} +{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-service.yaml b/chart/templates/rpc-server/rpc-server-service.yaml new file mode 100644 index 0000000000000..68da246b0a7fd --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-service.yaml @@ -0,0 +1,59 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################ +## Airflow rpc-server Service +################################# +{{- if .Values.rpcServer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }}-rpc-server + labels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.rpcServer.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.rpcServer.service.type }} + selector: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + ports: + {{ range .Values.rpcServer.service.ports }} + - + {{- range $key, $val := . }} + {{ $key }}: {{ tpl (toString $val) $ }} + {{- end }} + {{- end }} + {{- if .Values.rpcServer.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.rpcServer.service.loadBalancerIP }} + {{- end }} + {{- if .Values.rpcServer.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.rpcServer.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-serviceaccount.yaml b/chart/templates/rpc-server/rpc-server-serviceaccount.yaml new file mode 100644 index 0000000000000..490311e091f43 --- /dev/null +++ b/chart/templates/rpc-server/rpc-server-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +###################################### +## Airflow rpc-server ServiceAccount +###################################### +{{- if and .Values.rpcServer.enabled .Values.rpcServer.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.rpcServer.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "rpcServer.serviceAccountName" . }} + labels: + tier: airflow + component: rpc-server + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or (.Values.labels) (.Values.rpcServer.labels) }} + {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with .Values.rpcServer.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 360ba565a5a2a..33fb5316ddabc 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -4235,6 +4235,745 @@ } } }, + "rpcServer": { + "description": "Airflow RPC server settings.", + "type": "object", + "x-docsSection": "RPC Server", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable RPC server", + "type": "boolean", + "default": true + }, + "baseUrl": { + "description": "RPC server base URL", + "type": "string", + "default": "" + }, + "configMapAnnotations": { + "description": "Extra annotations to apply to the RPC server configmap.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "hostAliases": { + "description": "HostAliases for the RPC server pod.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.1", + "hostnames": [ + "foo.local" + ] + }, + { + "ip": "10.1.2.3", + "hostnames": [ + "foo.remote" + ] + } + ] + }, + "allowPodLogReading": { + "description": "Allow RPC server to read k8s pod logs. Useful when you don't have an external log store.", + "type": "boolean", + "default": true + }, + "livenessProbe": { + "description": "Liveness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "RPC server Liveness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "RPC server Liveness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "RPC server Liveness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "RPC server Liveness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "RPC server Liveness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "initialDelaySeconds": { + "description": "RPC server Readiness probe initial delay.", + "type": "integer", + "default": 15 + }, + "timeoutSeconds": { + "description": "RPC server Readiness probe timeout seconds.", + "type": "integer", + "default": 5 + }, + "failureThreshold": { + "description": "RPC server Readiness probe failure threshold.", + "type": "integer", + "default": 5 + }, + "periodSeconds": { + "description": "RPC server Readiness probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "RPC server Readiness probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "startupProbe": { + "description": "Startup probe configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "timeoutSeconds": { + "description": "RPC server Startup probe timeout seconds.", + "type": "integer", + "default": 20 + }, + "failureThreshold": { + "description": "RPC server Startup probe failure threshold.", + "type": "integer", + "default": 6 + }, + "periodSeconds": { + "description": "RPC server Startup probe period seconds.", + "type": "integer", + "default": 10 + }, + "scheme": { + "description": "RPC server Startup probe scheme.", + "type": "string", + "default": "HTTP" + } + } + }, + "replicas": { + "description": "How many Airflow RPC server replicas should run.", + "type": "integer", + "default": 1 + }, + "revisionHistoryLimit": { + "description": "Number of old replicasets to retain.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": null + }, + "command": { + "description": "Command to use when running the Airflow RPC server (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash" + ] + }, + "args": { + "description": "Args to use when running the Airflow RPC server (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "-c", + "exec airflow internal-api" + ] + }, + "strategy": { + "description": "Specifies the strategy used to replace old Pods by new ones.", + "type": [ + "null", + "object" + ], + "default": null + }, + "serviceAccount": { + "description": "Create ServiceAccount.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods", + "type": "boolean", + "default": true + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": "boolean", + "default": true + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the RPC server Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, + "podDisruptionBudget": { + "description": "RPC server pod disruption budget.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean", + "default": false + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for RPC server.", + "type": [ + "integer", + "string" + ], + "default": 1 + }, + "minAvailable": { + "description": "Min available pods for RPC server.", + "type": [ + "integer", + "string" + ], + "default": 1 + } + } + } + } + }, + "extraNetworkPolicies": { + "description": "Additional NetworkPolicies as needed (Deprecated - renamed to `RPC server.networkPolicy.ingress.from`).", + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "default": [] + }, + "networkPolicy": { + "description": "RPC server NetworkPolicy configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "ingress": { + "description": "RPC server NetworkPolicyingress configuration", + "type": "object", + "additionalProperties": false, + "properties": { + "from": { + "description": "Peers for RPC server NetworkPolicyingress.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + }, + "ports": { + "description": "Ports for RPC server NetworkPolicyingress (if `from` is set).", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + }, + "default": [ + { + "port": "{{ .Values.ports.rpcServer }}" + } + ], + "examples": [ + { + "port": 9080 + } + ] + } + } + } + } + }, + "securityContext": { + "description": "Security context for the RPC server job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the RPC server. If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the RPC server. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "pod": { + "description": "Pod security context definition for the RPC server.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 50000, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, + "container": { + "description": "Container security context definition for the RPC server.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + }, + "resources": { + "description": "Resources for RPC server pods.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "defaultUser": { + "description": "Optional default Airflow user information", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable default user creation.", + "type": "boolean", + "x-docsSection": "Common", + "default": true + }, + "role": { + "description": "Default user role.", + "type": "string", + "default": "Admin" + }, + "username": { + "description": "Default user username.", + "type": "string", + "default": "admin" + }, + "email": { + "description": "Default user email address.", + "type": "string", + "default": "admin@example.com" + }, + "firstName": { + "description": "Default user firstname.", + "type": "string", + "default": "admin" + }, + "lastName": { + "description": "Default user lastname.", + "type": "string", + "default": "user" + }, + "password": { + "description": "Default user password.", + "type": "string", + "default": "admin" + } + } + }, + "extraContainers": { + "description": "Launch additional containers into RPC server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into RPC server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Mount additional volumes into RPC server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into RPC server.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "RPC serverConfig": { + "description": "This string (can be templated) will be mounted into the Airflow RPC server as a custom `RPC server_config.py`. You can bake a `RPC server_config.py` in to your image instead or specify a configmap containing the RPC server_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "from airflow import configuration as conf\n\n# The SQLAlchemy connection string.\nSQLALCHEMY_DATABASE_URI = conf.get('database', 'SQL_ALCHEMY_CONN')\n\n# Flask-WTF flag for CSRF\nCSRF_ENABLED = True" + ] + }, + "RPC serverConfigConfigMapName": { + "description": "The configmap name containing the RPC server_config.py.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null, + "examples": [ + "my-RPC server-configmap" + ] + }, + "service": { + "description": "RPC server Service configuration.", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "description": "RPC server Service type.", + "type": "string", + "default": "ClusterIP" + }, + "annotations": { + "description": "Annotations for the RPC server Service.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "ports": { + "description": "Ports for the RPC server Service.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "port": { + "type": [ + "string", + "integer" + ] + }, + "targetPort": { + "type": [ + "string", + "integer" + ] + }, + "nodePort": { + "type": [ + "string", + "integer" + ] + }, + "protocol": { + "type": "string" + } + } + }, + "default": [ + { + "name": "rpc-server", + "port": "{{ .Values.ports.rpcServer }}" + } + ], + "examples": [ + { + "name": "rpc-server", + "port": 9080, + "targetPort": "rpc-server" + }, + { + "name": "only_sidecar", + "port": 9080, + "targetPort": 8888 + } + ] + }, + "loadBalancerIP": { + "description": "RPC server Service loadBalancerIP.", + "type": [ + "string", + "null" + ], + "default": null + }, + "loadBalancerSourceRanges": { + "description": "RPC server Service ``loadBalancerSourceRanges``.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + "10.123.0.0/16" + ] + } + } + }, + "nodeSelector": { + "description": "Select certain nodes for RPC server pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "priorityClassName": { + "description": "Specify priority for RPC server pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for RPC server pods.", + "type": "object", + "default": "See values.yaml", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for RPC server pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for RPC server pods.", + "type": "array", + "default": [], + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "annotations": { + "description": "Annotations to add to the RPC server deployment", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the RPC server pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the RPC server objects and pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "waitForMigrations": { + "description": "wait-for-airflow-migrations init container.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Enable wait-for-airflow-migrations init container.", + "type": "boolean", + "default": true + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + }, + "securityContexts": { + "description": "Security context definition for the wait for migrations. If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the wait for migrations.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "env": { + "description": "Add additional env vars to RPC server.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } + } + } + }, "webserver": { "description": "Airflow webserver settings.", "type": "object", @@ -6759,6 +7498,11 @@ "type": "integer", "default": 8080 }, + "rpcServer": { + "description": "RPC server port.", + "type": "integer", + "default": 9080 + }, "workerLogs": { "description": "Worker logs port.", "type": "integer", diff --git a/chart/values.yaml b/chart/values.yaml index 595e3e75c42f6..1908684aa8ebf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -185,6 +185,50 @@ ingress: # Http paths to add to the web Ingress after the default path succeedingPaths: [] + # Configs for the Ingress of the RPC server Service + rpcServer: + # Enable RPC server ingress resource + enabled: false + + # Annotations for the RPC server Ingress + annotations: {} + + # The path for the RPC server Ingress + path: "/" + + # The pathType for the above path (used only with Kubernetes v1.19 and above) + pathType: "ImplementationSpecific" + + # The hostname for the RPC server Ingress (Deprecated - renamed to `ingress.RPC server.hosts`) + host: "" + + # The hostnames or hosts configuration for the RPC server Ingress + hosts: [] + # # The hostname for the RPC server Ingress (can be templated) + # - name: "" + # # configs for RPC server Ingress TLS + # tls: + # # Enable TLS termination for the RPC server Ingress + # enabled: false + # # the name of a pre-created Secret containing a TLS private key and certificate + # secretName: "" + + # The Ingress Class for the RPC server Ingress (used only with Kubernetes v1.19 and above) + ingressClassName: "" + + # configs for RPC server Ingress TLS (Deprecated - renamed to `ingress.RPC server.hosts[*].tls`) + tls: + # Enable TLS termination for the RPC server Ingress + enabled: false + # the name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # HTTP paths to add to the RPC server Ingress before the default path + precedingPaths: [] + + # Http paths to add to the RPC server Ingress after the default path + succeedingPaths: [] + # Configs for the Ingress of the flower Service flower: # Enable web ingress resource @@ -1156,6 +1200,84 @@ migrateDatabaseJob: useHelmHooks: true applyCustomEnv: true +rpcServer: + enabled: true + baseUrl: "" + + # Command to use when running the Airflow rpc server (templated). + command: + - "bash" + # Args to use when running the Airflow rpc server (templated). + args: ["-c", "exec airflow internal-api"] + env: [] + serviceAccount: + # default value is true + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: true + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the release name + name: ~ + + # Annotations to add to webserver kubernetes service account. + annotations: {} + service: + type: ClusterIP + ## service annotations + annotations: {} + ports: + - name: rpc-server + port: "{{ .Values.ports.rpcServer }}" + + loadBalancerIP: ~ + ## Limit load balancer source ips to list of CIDRs + # loadBalancerSourceRanges: + # - "10.123.0.0/16" + loadBalancerSourceRanges: [] + + podDisruptionBudget: + enabled: false + + # PDB configuration + config: + # minAvailable and maxUnavailable are mutually exclusive + maxUnavailable: 1 + # minAvailable: 1 + + # Detailed default security contexts for webserver deployments for container and pod level + securityContexts: + pod: {} + container: {} + + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: true + env: [] + # Detailed default security context for waitForMigrations for container level + securityContexts: + container: {} + + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + readinessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 5 + failureThreshold: 5 + periodSeconds: 10 + scheme: HTTP + + startupProbe: + timeoutSeconds: 20 + failureThreshold: 6 + periodSeconds: 10 + scheme: HTTP + # Airflow webserver settings webserver: enabled: true @@ -2233,6 +2355,7 @@ elasticsearch: ports: flowerUI: 5555 airflowUI: 8080 + rpcServer: 9080 workerLogs: 8793 triggererLogs: 8794 redisDB: 6379 From 6553e5130f41ac9ca94e4475d6cd2e5861d221fd Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:59:58 -0700 Subject: [PATCH 02/21] add tests --- .../rpc-server/rpc-server-deployment.yaml | 2 +- chart/values.yaml | 13 + helm_tests/webserver/test_rpc_server.py | 917 ++++++++++++++++++ 3 files changed, 931 insertions(+), 1 deletion(-) create mode 100644 helm_tests/webserver/test_rpc_server.py diff --git a/chart/templates/rpc-server/rpc-server-deployment.yaml b/chart/templates/rpc-server/rpc-server-deployment.yaml index 78813ce91228c..fb900b4963112 100644 --- a/chart/templates/rpc-server/rpc-server-deployment.yaml +++ b/chart/templates/rpc-server/rpc-server-deployment.yaml @@ -237,7 +237,7 @@ spec: {{- include "git_sync_container" . | nindent 8 }} {{- end }} {{- if .Values.rpcServer.extraContainers }} - {{- toYaml .Values.rpcServer.extraContainers | nindent 8 }} + {{- tpl (toYaml .Values.rpcServer.extraContainers) . | nindent 8 }} {{- end }} volumes: - name: config diff --git a/chart/values.yaml b/chart/values.yaml index 1908684aa8ebf..298e871a91597 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1258,6 +1258,19 @@ rpcServer: securityContexts: container: {} + # Launch additional containers into the flower pods. + extraContainers: [] + + # Additional network policies as needed (Deprecated - renamed to `webserver.networkPolicy.ingress.from`) + extraNetworkPolicies: [] + networkPolicy: + ingress: + # Peers for webserver NetworkPolicy ingress + from: [] + # Ports for webserver NetworkPolicy ingress (if `from` is set) + ports: + - port: "{{ .Values.ports.rpcServer }}" + livenessProbe: initialDelaySeconds: 15 timeoutSeconds: 5 diff --git a/helm_tests/webserver/test_rpc_server.py b/helm_tests/webserver/test_rpc_server.py new file mode 100644 index 0000000000000..d2a7de7e9d168 --- /dev/null +++ b/helm_tests/webserver/test_rpc_server.py @@ -0,0 +1,917 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import jmespath +import pytest + +from tests.charts.helm_template_generator import render_chart + + +class TestRPCServerDeployment: + """Tests rpc-server deployment.""" + + def test_can_be_disabled(self): + """ + RPC server should be able to be disabled if the users desires. + + For example, user may be disabled when using rpc-server and having it deployed on another host. + """ + docs = render_chart( + values={"rpcServer": {"enabled": False}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert 0 == len(docs) + + def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(self): + docs = render_chart( + values={"rpcServer": {"baseUrl": "https://example.com:21222/mypath/path"}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert {"name": "Host", "value": "example.com"} in jmespath.search( + "spec.template.spec.containers[0].livenessProbe.httpGet.httpHeaders", docs[0] + ) + assert {"name": "Host", "value": "example.com"} in jmespath.search( + "spec.template.spec.containers[0].readinessProbe.httpGet.httpHeaders", docs[0] + ) + assert {"name": "Host", "value": "example.com"} in jmespath.search( + "spec.template.spec.containers[0].startupProbe.httpGet.httpHeaders", docs[0] + ) + + def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): + docs = render_chart( + values={"rpcServer": {"baseUrl": "https://example.com:21222/mypath/path"}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert ( + jmespath.search("spec.template.spec.containers[0].livenessProbe.httpGet.path", docs[0]) + == "/mypath/path/health" + ) + assert ( + jmespath.search("spec.template.spec.containers[0].readinessProbe.httpGet.path", docs[0]) + == "/mypath/path/health" + ) + assert ( + jmespath.search("spec.template.spec.containers[0].startupProbe.httpGet.path", docs[0]) + == "/mypath/path/health" + ) + + @pytest.mark.parametrize( + "revision_history_limit, global_revision_history_limit", + [(8, 10), (10, 8), (8, None), (None, 10), (None, None)], + ) + def test_revision_history_limit(self, revision_history_limit, global_revision_history_limit): + values = {"rpcServer": {}} + if revision_history_limit: + values["rpcServer"]["revisionHistoryLimit"] = revision_history_limit + if global_revision_history_limit: + values["revisionHistoryLimit"] = global_revision_history_limit + docs = render_chart( + values=values, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + expected_result = revision_history_limit if revision_history_limit else global_revision_history_limit + assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected_result + + @pytest.mark.parametrize("values", [{"config": {"rpcServer": {"base_url": ""}}}, {}]) + def test_should_not_contain_host_header(self, values): + print(values) + docs = render_chart(values=values, show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + + assert ( + jmespath.search("spec.template.spec.containers[0].livenessProbe.httpGet.httpHeaders", docs[0]) + is None + ) + assert ( + jmespath.search("spec.template.spec.containers[0].readinessProbe.httpGet.httpHeaders", docs[0]) + is None + ) + assert ( + jmespath.search("spec.template.spec.containers[0].startupProbe.httpGet.httpHeaders", docs[0]) + is None + ) + + def test_should_use_templated_base_url_for_probes(self): + docs = render_chart( + values={ + "rpcServer": { + "baseUrl": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path" + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + container = jmespath.search("spec.template.spec.containers[0]", docs[0]) + + assert {"name": "Host", "value": "release-name.com"} in jmespath.search( + "livenessProbe.httpGet.httpHeaders", container + ) + assert {"name": "Host", "value": "release-name.com"} in jmespath.search( + "readinessProbe.httpGet.httpHeaders", container + ) + assert {"name": "Host", "value": "release-name.com"} in jmespath.search( + "startupProbe.httpGet.httpHeaders", container + ) + assert "/mypath/release-name/path/health" == jmespath.search("livenessProbe.httpGet.path", container) + assert "/mypath/release-name/path/health" == jmespath.search("readinessProbe.httpGet.path", container) + assert "/mypath/release-name/path/health" == jmespath.search("startupProbe.httpGet.path", container) + + def test_should_add_scheme_to_liveness_and_readiness_and_startup_probes(self): + docs = render_chart( + values={ + "rpcServer": { + "livenessProbe": {"scheme": "HTTPS"}, + "readinessProbe": {"scheme": "HTTPS"}, + "startupProbe": {"scheme": "HTTPS"}, + } + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "HTTPS" in jmespath.search( + "spec.template.spec.containers[0].livenessProbe.httpGet.scheme", docs[0] + ) + assert "HTTPS" in jmespath.search( + "spec.template.spec.containers[0].readinessProbe.httpGet.scheme", docs[0] + ) + assert "HTTPS" in jmespath.search( + "spec.template.spec.containers[0].startupProbe.httpGet.scheme", docs[0] + ) + + def test_should_add_extra_containers(self): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "rpcServer": { + "extraContainers": [ + {"name": "{{.Chart.Name}}", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert jmespath.search("spec.template.spec.containers[-1]", docs[0]) == { + "name": "airflow", + "image": "test-registry/test-repo:test-tag", + } + + def test_should_add_extraEnvs(self): + docs = render_chart( + values={ + "rpcServer": { + "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( + "spec.template.spec.containers[0].env", docs[0] + ) + + def test_should_add_extra_volume_and_extra_volume_mount(self): + docs = render_chart( + values={ + "rpcServer": { + "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "test-volume-airflow" == jmespath.search("spec.template.spec.volumes[-1].name", docs[0]) + assert "test-volume-airflow" == jmespath.search( + "spec.template.spec.containers[0].volumeMounts[-1].name", docs[0] + ) + assert "test-volume-airflow" == jmespath.search( + "spec.template.spec.initContainers[0].volumeMounts[-1].name", docs[0] + ) + + def test_should_add_global_volume_and_global_volume_mount(self): + docs = render_chart( + values={ + "volumes": [{"name": "test-volume", "emptyDir": {}}], + "volumeMounts": [{"name": "test-volume", "mountPath": "/opt/test"}], + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "test-volume" == jmespath.search("spec.template.spec.volumes[-1].name", docs[0]) + assert "test-volume" == jmespath.search( + "spec.template.spec.containers[0].volumeMounts[-1].name", docs[0] + ) + + def test_should_add_extraEnvs_to_wait_for_migration_container(self): + docs = render_chart( + values={ + "rpcServer": { + "waitForMigrations": { + "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + }, + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( + "spec.template.spec.initContainers[0].env", docs[0] + ) + + @pytest.mark.parametrize( + "airflow_version, expected_arg", + [ + ("2.0.0", ["airflow", "db", "check-migrations", "--migration-wait-timeout=60"]), + ("2.1.0", ["airflow", "db", "check-migrations", "--migration-wait-timeout=60"]), + ("1.10.2", ["python", "-c"]), + ], + ) + def test_wait_for_migration_airflow_version(self, airflow_version, expected_arg): + docs = render_chart( + values={ + "airflowVersion": airflow_version, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + # Don't test the full string, just the length of the expect matches + actual = jmespath.search("spec.template.spec.initContainers[0].args", docs[0]) + assert expected_arg == actual[: len(expected_arg)] + + def test_disable_wait_for_migration(self): + docs = render_chart( + values={ + "rpcServer": { + "waitForMigrations": {"enabled": False}, + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + actual = jmespath.search( + "spec.template.spec.initContainers[?name=='wait-for-airflow-migrations']", docs[0] + ) + assert actual is None + + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "rpcServer": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) + + def test_should_add_component_specific_labels(self): + docs = render_chart( + values={ + "rpcServer": { + "labels": {"test_label": "test_label_value"}, + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "test_label" in jmespath.search("spec.template.metadata.labels", docs[0]) + assert jmespath.search("spec.template.metadata.labels", docs[0])["test_label"] == "test_label_value" + + def test_should_create_valid_affinity_tolerations_and_node_selector(self): + docs = render_chart( + values={ + "rpcServer": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + "nodeSelector": {"diskType": "ssd"}, + } + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "Deployment" == jmespath.search("kind", docs[0]) + assert "foo" == jmespath.search( + "spec.template.spec.affinity.nodeAffinity." + "requiredDuringSchedulingIgnoredDuringExecution." + "nodeSelectorTerms[0]." + "matchExpressions[0]." + "key", + docs[0], + ) + assert "ssd" == jmespath.search( + "spec.template.spec.nodeSelector.diskType", + docs[0], + ) + assert "dynamic-pods" == jmespath.search( + "spec.template.spec.tolerations[0].key", + docs[0], + ) + + def test_should_create_default_affinity(self): + docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + + assert {"component": "rpc-server"} == jmespath.search( + "spec.template.spec.affinity.podAntiAffinity." + "preferredDuringSchedulingIgnoredDuringExecution[0]." + "podAffinityTerm.labelSelector.matchLabels", + docs[0], + ) + + def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): + """When given both global and rpc-server affinity etc, rpc-server affinity etc is used.""" + expected_affinity = { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + } + expected_topology_spread_constraints = { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + docs = render_chart( + values={ + "rpcServer": { + "affinity": expected_affinity, + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + "topologySpreadConstraints": [expected_topology_spread_constraints], + "nodeSelector": {"type": "ssd"}, + }, + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 1, + "preference": { + "matchExpressions": [ + {"key": "not-me", "operator": "In", "values": ["true"]}, + ] + }, + } + ] + } + }, + "tolerations": [ + {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "not-me", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + "nodeSelector": {"type": "not-me"}, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert expected_affinity == jmespath.search("spec.template.spec.affinity", docs[0]) + assert "ssd" == jmespath.search( + "spec.template.spec.nodeSelector.type", + docs[0], + ) + tolerations = jmespath.search("spec.template.spec.tolerations", docs[0]) + assert 1 == len(tolerations) + assert "dynamic-pods" == tolerations[0]["key"] + assert expected_topology_spread_constraints == jmespath.search( + "spec.template.spec.topologySpreadConstraints[0]", docs[0] + ) + + def test_scheduler_name(self): + docs = render_chart( + values={"schedulerName": "airflow-scheduler"}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "airflow-scheduler" == jmespath.search( + "spec.template.spec.schedulerName", + docs[0], + ) + + @pytest.mark.parametrize( + "log_persistence_values, expected_claim_name", + [ + ({"enabled": False}, None), + ({"enabled": True}, "release-name-logs"), + ({"enabled": True, "existingClaim": "test-claim"}, "test-claim"), + ], + ) + def test_logs_persistence_adds_volume_and_mount(self, log_persistence_values, expected_claim_name): + docs = render_chart( + values={"logs": {"persistence": log_persistence_values}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + if expected_claim_name: + assert { + "name": "logs", + "persistentVolumeClaim": {"claimName": expected_claim_name}, + } in jmespath.search("spec.template.spec.volumes", docs[0]) + assert { + "name": "logs", + "mountPath": "/opt/airflow/logs", + } in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + else: + assert "logs" not in [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])] + assert "logs" not in [ + v["name"] for v in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + ] + + def test_config_volumes(self): + docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + + # default config + assert { + "name": "config", + "mountPath": "/opt/airflow/airflow.cfg", + "readOnly": True, + "subPath": "airflow.cfg", + } in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + + def test_rpc_server_resources_are_configurable(self): + docs = render_chart( + values={ + "rpcServer": { + "resources": { + "limits": {"cpu": "200m", "memory": "128Mi"}, + "requests": {"cpu": "300m", "memory": "169Mi"}, + } + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + assert "128Mi" == jmespath.search("spec.template.spec.containers[0].resources.limits.memory", docs[0]) + assert "200m" == jmespath.search("spec.template.spec.containers[0].resources.limits.cpu", docs[0]) + + assert "169Mi" == jmespath.search( + "spec.template.spec.containers[0].resources.requests.memory", docs[0] + ) + assert "300m" == jmespath.search("spec.template.spec.containers[0].resources.requests.cpu", docs[0]) + + # initContainer wait-for-airflow-migrations + assert "128Mi" == jmespath.search( + "spec.template.spec.initContainers[0].resources.limits.memory", docs[0] + ) + assert "200m" == jmespath.search("spec.template.spec.initContainers[0].resources.limits.cpu", docs[0]) + + assert "169Mi" == jmespath.search( + "spec.template.spec.initContainers[0].resources.requests.memory", docs[0] + ) + assert "300m" == jmespath.search( + "spec.template.spec.initContainers[0].resources.requests.cpu", docs[0] + ) + + def test_rpc_server_security_contexts_are_configurable(self): + docs = render_chart( + values={ + "rpcServer": { + "securityContexts": { + "pod": { + "fsGroup": 1000, + "runAsGroup": 1001, + "runAsNonRoot": True, + "runAsUser": 2000, + }, + "container": { + "allowPrivilegeEscalation": False, + "readOnlyRootFilesystem": True, + }, + } + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + assert {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True} == jmespath.search( + "spec.template.spec.containers[0].securityContext", docs[0] + ) + + assert { + "runAsUser": 2000, + "runAsGroup": 1001, + "fsGroup": 1000, + "runAsNonRoot": True, + } == jmespath.search("spec.template.spec.securityContext", docs[0]) + + def test_rpc_server_security_context_legacy(self): + docs = render_chart( + values={ + "rpcServer": { + "securityContext": { + "fsGroup": 1000, + "runAsGroup": 1001, + "runAsNonRoot": True, + "runAsUser": 2000, + } + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert { + "runAsUser": 2000, + "runAsGroup": 1001, + "fsGroup": 1000, + "runAsNonRoot": True, + } == jmespath.search("spec.template.spec.securityContext", docs[0]) + + def test_rpc_server_resources_are_not_added_by_default(self): + docs = render_chart( + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == {} + assert jmespath.search("spec.template.spec.initContainers[0].resources", docs[0]) == {} + + @pytest.mark.parametrize( + "airflow_version, expected_strategy", + [ + ("2.0.2", {"type": "RollingUpdate", "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}}), + ("1.10.14", {"type": "Recreate"}), + ("1.9.0", {"type": "Recreate"}), + ("2.1.0", {"type": "RollingUpdate", "rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0}}), + ], + ) + def test_default_update_strategy(self, airflow_version, expected_strategy): + docs = render_chart( + values={"airflowVersion": airflow_version}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert jmespath.search("spec.strategy", docs[0]) == expected_strategy + + def test_update_strategy(self): + expected_strategy = {"type": "RollingUpdate", "rollingUpdate": {"maxUnavailable": 1}} + docs = render_chart( + values={"rpcServer": {"strategy": expected_strategy}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert jmespath.search("spec.strategy", docs[0]) == expected_strategy + + def test_no_airflow_local_settings(self): + docs = render_chart( + values={"airflowLocalSettings": None}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + assert "airflow_local_settings.py" not in str(volume_mounts) + volume_mounts_init = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + assert "airflow_local_settings.py" not in str(volume_mounts_init) + + def test_airflow_local_settings(self): + docs = render_chart( + values={"airflowLocalSettings": "# Well hello!"}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + volume_mount = { + "name": "config", + "mountPath": "/opt/airflow/config/airflow_local_settings.py", + "subPath": "airflow_local_settings.py", + "readOnly": True, + } + assert volume_mount in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + assert volume_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0]) + + def test_default_command_and_args(self): + docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + + assert jmespath.search("spec.template.spec.containers[0].command", docs[0]) == ["bash"] + assert ["-c", "exec airflow internal-api"] == jmespath.search( + "spec.template.spec.containers[0].args", docs[0] + ) + + @pytest.mark.parametrize("command", [None, ["custom", "command"]]) + @pytest.mark.parametrize("args", [None, ["custom", "args"]]) + def test_command_and_args_overrides(self, command, args): + docs = render_chart( + values={"rpcServer": {"command": command, "args": args}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert command == jmespath.search("spec.template.spec.containers[0].command", docs[0]) + assert args == jmespath.search("spec.template.spec.containers[0].args", docs[0]) + + def test_command_and_args_overrides_are_templated(self): + docs = render_chart( + values={"rpcServer": {"command": ["{{ .Release.Name }}"], "args": ["{{ .Release.Service }}"]}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert ["release-name"] == jmespath.search("spec.template.spec.containers[0].command", docs[0]) + assert ["Helm"] == jmespath.search("spec.template.spec.containers[0].args", docs[0]) + + def test_should_add_component_specific_annotations(self): + docs = render_chart( + values={ + "rpcServer": { + "annotations": {"test_annotation": "test_annotation_value"}, + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + assert "annotations" in jmespath.search("metadata", docs[0]) + assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value" + + def test_rpc_server_pod_hostaliases(self): + docs = render_chart( + values={ + "rpcServer": { + "hostAliases": [{"ip": "127.0.0.1", "hostnames": ["foo.local"]}], + }, + }, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) + + assert "127.0.0.1" == jmespath.search("spec.template.spec.hostAliases[0].ip", docs[0]) + assert "foo.local" == jmespath.search("spec.template.spec.hostAliases[0].hostnames[0]", docs[0]) + + +class TestRPCServerService: + """Tests rpc-server service.""" + + def test_default_service(self): + docs = render_chart(show_only=["templates/rpc-server/rpc-server-service.yaml"]) + + assert "release-name-rpc-server" == jmespath.search("metadata.name", docs[0]) + assert jmespath.search("metadata.annotations", docs[0]) is None + assert {"tier": "airflow", "component": "rpc-server", "release": "release-name"} == jmespath.search( + "spec.selector", docs[0] + ) + assert "ClusterIP" == jmespath.search("spec.type", docs[0]) + assert {"name": "rpc-server", "port": 9080} in jmespath.search("spec.ports", docs[0]) + + def test_overrides(self): + docs = render_chart( + values={ + "ports": {"rpcServer": 9000}, + "rpcServer": { + "service": { + "type": "LoadBalancer", + "loadBalancerIP": "127.0.0.1", + "annotations": {"foo": "bar"}, + "loadBalancerSourceRanges": ["10.123.0.0/16"], + } + }, + }, + show_only=["templates/rpc-server/rpc-server-service.yaml"], + ) + + assert {"foo": "bar"} == jmespath.search("metadata.annotations", docs[0]) + assert "LoadBalancer" == jmespath.search("spec.type", docs[0]) + assert {"name": "rpc-server", "port": 9000} in jmespath.search("spec.ports", docs[0]) + assert "127.0.0.1" == jmespath.search("spec.loadBalancerIP", docs[0]) + assert ["10.123.0.0/16"] == jmespath.search("spec.loadBalancerSourceRanges", docs[0]) + + @pytest.mark.parametrize( + "ports, expected_ports", + [ + ([{"port": 8888}], [{"port": 8888}]), # name is optional with a single port + ( + [{"name": "{{ .Release.Name }}", "protocol": "UDP", "port": "{{ .Values.ports.rpcServer }}"}], + [{"name": "release-name", "protocol": "UDP", "port": 8080}], + ), + ([{"name": "only_sidecar", "port": "{{ int 9000 }}"}], [{"name": "only_sidecar", "port": 9000}]), + ( + [ + {"name": "airflow-ui", "port": "{{ .Values.ports.rpcServer }}"}, + {"name": "sidecar", "port": 80, "targetPort": "sidecar"}, + ], + [ + {"name": "airflow-ui", "port": 8080}, + {"name": "sidecar", "port": 80, "targetPort": "sidecar"}, + ], + ), + ], + ) + def test_ports_overrides(self, ports, expected_ports): + docs = render_chart( + values={ + "rpcServer": {"service": {"ports": ports}}, + }, + show_only=["templates/rpc-server/rpc-server-service.yaml"], + ) + + assert expected_ports == jmespath.search("spec.ports", docs[0]) + + def test_should_add_component_specific_labels(self): + docs = render_chart( + values={ + "rpcServer": { + "labels": {"test_label": "test_label_value"}, + }, + }, + show_only=["templates/rpc-server/rpc-server-service.yaml"], + ) + assert "test_label" in jmespath.search("metadata.labels", docs[0]) + assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + + @pytest.mark.parametrize( + "ports, expected_ports", + [ + ( + [{"nodePort": "31000", "port": "8080"}], + [{"nodePort": 31000, "port": 8080}], + ), + ( + [{"port": "8080"}], + [{"port": 8080}], + ), + ], + ) + def test_nodeport_service(self, ports, expected_ports): + docs = render_chart( + values={ + "rpcServer": { + "service": { + "type": "NodePort", + "ports": ports, + } + }, + }, + show_only=["templates/rpc-server/rpc-server-service.yaml"], + ) + + assert "NodePort" == jmespath.search("spec.type", docs[0]) + assert expected_ports == jmespath.search("spec.ports", docs[0]) + + +class TestRPCServerNetworkPolicy: + """Tests rpc-server network policy.""" + + def test_off_by_default(self): + docs = render_chart( + show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], + ) + assert 0 == len(docs) + + def test_defaults(self): + docs = render_chart( + values={ + "networkPolicies": {"enabled": True}, + "rpcServer": { + "networkPolicy": { + "ingress": { + "from": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] + } + } + }, + }, + show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], + ) + + assert 1 == len(docs) + assert "NetworkPolicy" == docs[0]["kind"] + assert [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] == jmespath.search( + "spec.ingress[0].from", docs[0] + ) + assert jmespath.search("spec.ingress[0].ports", docs[0]) == [{"port": 9080}] + + @pytest.mark.parametrize( + "ports, expected_ports", + [ + ([{"port": "sidecar"}], [{"port": "sidecar"}]), + ( + [ + {"port": "{{ .Values.ports.rpcServer }}"}, + {"port": 80}, + ], + [ + {"port": 8080}, + {"port": 80}, + ], + ), + ], + ) + def test_ports_overrides(self, ports, expected_ports): + docs = render_chart( + values={ + "networkPolicies": {"enabled": True}, + "rpcServer": { + "networkPolicy": { + "ingress": { + "from": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}], + "ports": ports, + } + } + }, + }, + show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], + ) + + assert expected_ports == jmespath.search("spec.ingress[0].ports", docs[0]) + + def test_deprecated_from_param(self): + docs = render_chart( + values={ + "networkPolicies": {"enabled": True}, + "rpcServer": { + "extraNetworkPolicies": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] + }, + }, + show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], + ) + + assert [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] == jmespath.search( + "spec.ingress[0].from", docs[0] + ) + + def test_should_add_component_specific_labels(self): + docs = render_chart( + values={ + "networkPolicies": {"enabled": True}, + "rpcServer": { + "labels": {"test_label": "test_label_value"}, + }, + }, + show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], + ) + assert "test_label" in jmespath.search("metadata.labels", docs[0]) + assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + + +class TestRPCServerServiceAccount: + """Tests rpc-server service account.""" + + def test_should_add_component_specific_labels(self): + docs = render_chart( + values={ + "rpcServer": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + show_only=["templates/rpc-server/rpc-server-serviceaccount.yaml"], + ) + assert "test_label" in jmespath.search("metadata.labels", docs[0]) + assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + + def test_default_automount_service_account_token(self): + docs = render_chart( + values={ + "rpcServer": { + "serviceAccount": {"create": True}, + }, + }, + show_only=["templates/rpc-server/rpc-server-serviceaccount.yaml"], + ) + assert jmespath.search("automountServiceAccountToken", docs[0]) is True + + def test_overridden_automount_service_account_token(self): + docs = render_chart( + values={ + "rpcServer": { + "serviceAccount": {"create": True, "automountServiceAccountToken": False}, + }, + }, + show_only=["templates/rpc-server/rpc-server-serviceaccount.yaml"], + ) + assert jmespath.search("automountServiceAccountToken", docs[0]) is False From f24858fc23b6993a95028fad41a258e1a7a88f3e Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:07:38 -0700 Subject: [PATCH 03/21] last test fixes --- chart/values.yaml | 8 ++++++++ helm_tests/webserver/test_rpc_server.py | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 298e871a91597..e88fc179a58ad 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1271,6 +1271,14 @@ rpcServer: ports: - port: "{{ .Values.ports.rpcServer }}" + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + livenessProbe: initialDelaySeconds: 15 timeoutSeconds: 5 diff --git a/helm_tests/webserver/test_rpc_server.py b/helm_tests/webserver/test_rpc_server.py index d2a7de7e9d168..40068d4e0b872 100644 --- a/helm_tests/webserver/test_rpc_server.py +++ b/helm_tests/webserver/test_rpc_server.py @@ -62,15 +62,15 @@ def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): assert ( jmespath.search("spec.template.spec.containers[0].livenessProbe.httpGet.path", docs[0]) - == "/mypath/path/health" + == "/mypath/path/internal_api/v1/health" ) assert ( jmespath.search("spec.template.spec.containers[0].readinessProbe.httpGet.path", docs[0]) - == "/mypath/path/health" + == "/mypath/path/internal_api/v1/health" ) assert ( jmespath.search("spec.template.spec.containers[0].startupProbe.httpGet.path", docs[0]) - == "/mypath/path/health" + == "/mypath/path/internal_api/v1/health" ) @pytest.mark.parametrize( @@ -128,9 +128,15 @@ def test_should_use_templated_base_url_for_probes(self): assert {"name": "Host", "value": "release-name.com"} in jmespath.search( "startupProbe.httpGet.httpHeaders", container ) - assert "/mypath/release-name/path/health" == jmespath.search("livenessProbe.httpGet.path", container) - assert "/mypath/release-name/path/health" == jmespath.search("readinessProbe.httpGet.path", container) - assert "/mypath/release-name/path/health" == jmespath.search("startupProbe.httpGet.path", container) + assert "/mypath/release-name/path/internal_api/v1/health" == jmespath.search( + "livenessProbe.httpGet.path", container + ) + assert "/mypath/release-name/path/internal_api/v1/health" == jmespath.search( + "readinessProbe.httpGet.path", container + ) + assert "/mypath/release-name/path/internal_api/v1/health" == jmespath.search( + "startupProbe.httpGet.path", container + ) def test_should_add_scheme_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( From 3deee1b0f32158428591724d8f51936576da8717 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 4 Apr 2024 23:34:50 -0700 Subject: [PATCH 04/21] doc fix --- chart/values.schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/values.schema.json b/chart/values.schema.json index 33fb5316ddabc..0f78678c39df7 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -9,6 +9,7 @@ "Ports", "Database", "PgBouncer", + "RPC Server", "Scheduler", "Webserver", "Workers", From b6ca4d2d91879d2efc082ed636f9a8100642182a Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 5 Apr 2024 00:04:31 -0700 Subject: [PATCH 05/21] fix tests --- chart/values.yaml | 3 ++ .../airflow_aux/test_basic_helm_chart.py | 22 +++++++-- helm_tests/security/test_rbac.py | 47 +++++++++---------- helm_tests/webserver/test_rpc_server.py | 8 ++-- tests/charts/helm_template_generator.py | 1 + 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index e88fc179a58ad..0d2f54654a594 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1204,6 +1204,9 @@ rpcServer: enabled: true baseUrl: "" + # Labels specific to workers objects and pods + labels: {} + # Command to use when running the Airflow rpc server (templated). command: - "bash" diff --git a/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm_tests/airflow_aux/test_basic_helm_chart.py index 0db5936311916..d4a32b257b667 100644 --- a/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -27,7 +27,7 @@ from tests.charts.helm_template_generator import render_chart -OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 35 +OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 38 class TestBaseChartTest: @@ -66,6 +66,7 @@ def test_basic_deployments(self, version): ("ServiceAccount", "test-basic-create-user-job"), ("ServiceAccount", "test-basic-migrate-database-job"), ("ServiceAccount", "test-basic-redis"), + ("ServiceAccount", "test-basic-rpc-server"), ("ServiceAccount", "test-basic-scheduler"), ("ServiceAccount", "test-basic-statsd"), ("ServiceAccount", "test-basic-triggerer"), @@ -86,11 +87,13 @@ def test_basic_deployments(self, version): ("Service", "test-basic-postgresql-hl"), ("Service", "test-basic-postgresql"), ("Service", "test-basic-redis"), + ("Service", "test-basic-rpc-server"), ("Service", "test-basic-statsd"), ("Service", "test-basic-webserver"), ("Service", "test-basic-worker"), ("Deployment", "test-basic-scheduler"), ("Deployment", "test-basic-statsd"), + ("Deployment", "test-basic-rpc-server"), (self.default_trigger_obj(version), "test-basic-triggerer"), ("Deployment", "test-basic-webserver"), ("StatefulSet", "test-basic-postgresql"), @@ -104,7 +107,7 @@ def test_basic_deployments(self, version): if version == "default": expected.add(("Service", "test-basic-triggerer")) assert list_of_kind_names_tuples == expected - assert expected_object_count_in_basic_deployment == len(k8s_objects) + assert len(k8s_objects) == expected_object_count_in_basic_deployment for k8s_object in k8s_objects: labels = jmespath.search("metadata.labels", k8s_object) or {} if "helm.sh/chart" in labels: @@ -134,6 +137,7 @@ def test_basic_deployments_with_standard_naming(self): ("ServiceAccount", "test-basic-airflow-statsd"), ("ServiceAccount", "test-basic-airflow-triggerer"), ("ServiceAccount", "test-basic-airflow-webserver"), + ("ServiceAccount", "test-basic-airflow-rpc-server"), ("ServiceAccount", "test-basic-airflow-worker"), ("Secret", "test-basic-airflow-metadata"), ("Secret", "test-basic-broker-url"), @@ -151,12 +155,14 @@ def test_basic_deployments_with_standard_naming(self): ("Service", "test-basic-airflow-statsd"), ("Service", "test-basic-airflow-triggerer"), ("Service", "test-basic-airflow-webserver"), + ("Service", "test-basic-airflow-rpc-server"), ("Service", "test-basic-airflow-worker"), ("Service", "test-basic-postgresql"), ("Service", "test-basic-postgresql-hl"), ("Deployment", "test-basic-airflow-scheduler"), ("Deployment", "test-basic-airflow-statsd"), ("Deployment", "test-basic-airflow-webserver"), + ("Deployment", "test-basic-airflow-rpc-server"), ("StatefulSet", "test-basic-airflow-redis"), ("StatefulSet", "test-basic-airflow-worker"), ("StatefulSet", "test-basic-airflow-triggerer"), @@ -196,6 +202,7 @@ def test_basic_deployment_with_standalone_dag_processor(self, version): ("ServiceAccount", "test-basic-statsd"), ("ServiceAccount", "test-basic-triggerer"), ("ServiceAccount", "test-basic-dag-processor"), + ("ServiceAccount", "test-basic-rpc-server"), ("ServiceAccount", "test-basic-webserver"), ("ServiceAccount", "test-basic-worker"), ("Secret", "test-basic-metadata"), @@ -215,12 +222,14 @@ def test_basic_deployment_with_standalone_dag_processor(self, version): ("Service", "test-basic-redis"), ("Service", "test-basic-statsd"), ("Service", "test-basic-webserver"), + ("Service", "test-basic-rpc-server"), ("Service", "test-basic-worker"), ("Deployment", "test-basic-scheduler"), ("Deployment", "test-basic-statsd"), (self.default_trigger_obj(version), "test-basic-triggerer"), ("Deployment", "test-basic-dag-processor"), ("Deployment", "test-basic-webserver"), + ("Deployment", "test-basic-rpc-server"), ("StatefulSet", "test-basic-postgresql"), ("StatefulSet", "test-basic-redis"), ("StatefulSet", "test-basic-worker"), @@ -259,7 +268,7 @@ def test_basic_deployment_without_default_users(self, version): (k8s_object["kind"], k8s_object["metadata"]["name"]) for k8s_object in k8s_objects ] assert ("Job", "test-basic-create-user") not in list_of_kind_names_tuples - assert expected_object_count_in_basic_deployment - 2 == len(k8s_objects) + assert len(k8s_objects) == expected_object_count_in_basic_deployment - 2 @pytest.mark.parametrize("version", ["2.3.2", "2.4.0", "default"]) def test_basic_deployment_without_statsd(self, version): @@ -276,7 +285,7 @@ def test_basic_deployment_without_statsd(self, version): assert ("Service", "test-basic-statsd") not in list_of_kind_names_tuples assert ("Deployment", "test-basic-statsd") not in list_of_kind_names_tuples - assert expected_object_count_in_basic_deployment - 4 == len(k8s_objects) + assert len(k8s_objects) == expected_object_count_in_basic_deployment - 4 def test_network_policies_are_valid(self): k8s_objects = render_chart( @@ -511,7 +520,10 @@ def get_k8s_objs_with_image(obj: list[Any] | dict[str, Any]) -> list[dict[str, A image: str = obj["image"] if image.startswith(image_repo): # Make sure that a command is not specified - assert "command" not in obj + if obj["name"] == "rpc-server": + assert obj["command"] == ["bash"] + else: + assert "command" not in obj def test_unsupported_executor(self): with pytest.raises(CalledProcessError) as ex_ctx: diff --git a/helm_tests/security/test_rbac.py b/helm_tests/security/test_rbac.py index c305194c9ec46..fff2883f28263 100644 --- a/helm_tests/security/test_rbac.py +++ b/helm_tests/security/test_rbac.py @@ -34,6 +34,7 @@ ("Service", "test-rbac-postgresql"), ("Service", "test-rbac-statsd"), ("Service", "test-rbac-webserver"), + ("Service", "test-rbac-rpc-server"), ("Service", "test-rbac-flower"), ("Service", "test-rbac-pgbouncer"), ("Service", "test-rbac-redis"), @@ -41,6 +42,7 @@ ("Deployment", "test-rbac-scheduler"), ("Deployment", "test-rbac-statsd"), ("Deployment", "test-rbac-webserver"), + ("Deployment", "test-rbac-rpc-server"), ("Deployment", "test-rbac-flower"), ("Deployment", "test-rbac-pgbouncer"), ("StatefulSet", "test-rbac-postgresql"), @@ -68,6 +70,7 @@ ("ServiceAccount", "test-rbac-cleanup"), ("ServiceAccount", "test-rbac-scheduler"), ("ServiceAccount", "test-rbac-webserver"), + ("ServiceAccount", "test-rbac-rpc-server"), ("ServiceAccount", "test-rbac-worker"), ("ServiceAccount", "test-rbac-triggerer"), ("ServiceAccount", "test-rbac-pgbouncer"), @@ -79,31 +82,19 @@ ] CUSTOM_SERVICE_ACCOUNT_NAMES = ( - CUSTOM_SCHEDULER_NAME, - CUSTOM_WEBSERVER_NAME, - CUSTOM_WORKER_NAME, - CUSTOM_TRIGGERER_NAME, - CUSTOM_CLEANUP_NAME, - CUSTOM_FLOWER_NAME, - CUSTOM_PGBOUNCER_NAME, - CUSTOM_STATSD_NAME, - CUSTOM_CREATE_USER_JOBS_NAME, - CUSTOM_MIGRATE_DATABASE_JOBS_NAME, - CUSTOM_REDIS_NAME, - CUSTOM_POSTGRESQL_NAME, -) = ( - "TestScheduler", - "TestWebserver", - "TestWorker", - "TestTriggerer", - "TestCleanup", - "TestFlower", - "TestPGBouncer", - "TestStatsd", - "TestCreateUserJob", - "TestMigrateDatabaseJob", - "TestRedis", - "TestPostgresql", + (CUSTOM_SCHEDULER_NAME := "TestScheduler"), + (CUSTOM_WEBSERVER_NAME := "TestWebserver"), + (CUSTOM_RPC_SERVER_NAME := "TestRPCSserver"), + (CUSTOM_WORKER_NAME := "TestWorker"), + (CUSTOM_TRIGGERER_NAME := "TestTriggerer"), + (CUSTOM_CLEANUP_NAME := "TestCleanup"), + (CUSTOM_FLOWER_NAME := "TestFlower"), + (CUSTOM_PGBOUNCER_NAME := "TestPGBouncer"), + (CUSTOM_STATSD_NAME := "TestStatsd"), + (CUSTOM_CREATE_USER_JOBS_NAME := "TestCreateUserJob"), + (CUSTOM_MIGRATE_DATABASE_JOBS_NAME := "TestMigrateDatabaseJob"), + (CUSTOM_REDIS_NAME := "TestRedis"), + (CUSTOM_POSTGRESQL_NAME := "TestPostgresql"), ) @@ -150,6 +141,7 @@ def test_deployments_no_rbac_no_sa(self, version): "redis": {"serviceAccount": {"create": False}}, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, + "rpcServer": {"serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "statsd": {"serviceAccount": {"create": False}}, @@ -176,6 +168,7 @@ def test_deployments_no_rbac_with_sa(self, version): "cleanup": {"enabled": True}, "flower": {"enabled": True}, "pgbouncer": {"enabled": True}, + "rpcServer": {"enabled": True}, }, version=version, ), @@ -201,6 +194,7 @@ def test_deployments_with_rbac_no_sa(self, version): }, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, + "rpcServer": {"serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "flower": {"enabled": True, "serviceAccount": {"create": False}}, @@ -259,6 +253,7 @@ def test_service_account_custom_names(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, + "rpcServer": {"serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, @@ -295,6 +290,7 @@ def test_service_account_custom_names_in_objects(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, + "rpcServer": {"serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, @@ -337,6 +333,7 @@ def test_service_account_without_resource(self): "redis": {"enabled": False}, "flower": {"enabled": False}, "statsd": {"enabled": False}, + "rpcServer": {"enabled": False}, "webserver": {"defaultUser": {"enabled": False}}, }, ) diff --git a/helm_tests/webserver/test_rpc_server.py b/helm_tests/webserver/test_rpc_server.py index 40068d4e0b872..d6b6abd02108d 100644 --- a/helm_tests/webserver/test_rpc_server.py +++ b/helm_tests/webserver/test_rpc_server.py @@ -723,16 +723,16 @@ def test_overrides(self): ([{"port": 8888}], [{"port": 8888}]), # name is optional with a single port ( [{"name": "{{ .Release.Name }}", "protocol": "UDP", "port": "{{ .Values.ports.rpcServer }}"}], - [{"name": "release-name", "protocol": "UDP", "port": 8080}], + [{"name": "release-name", "protocol": "UDP", "port": 9080}], ), ([{"name": "only_sidecar", "port": "{{ int 9000 }}"}], [{"name": "only_sidecar", "port": 9000}]), ( [ - {"name": "airflow-ui", "port": "{{ .Values.ports.rpcServer }}"}, + {"name": "rpc-server", "port": "{{ .Values.ports.rpcServer }}"}, {"name": "sidecar", "port": 80, "targetPort": "sidecar"}, ], [ - {"name": "airflow-ui", "port": 8080}, + {"name": "rpc-server", "port": 9080}, {"name": "sidecar", "port": 80, "targetPort": "sidecar"}, ], ), @@ -746,7 +746,7 @@ def test_ports_overrides(self, ports, expected_ports): show_only=["templates/rpc-server/rpc-server-service.yaml"], ) - assert expected_ports == jmespath.search("spec.ports", docs[0]) + assert jmespath.search("spec.ports", docs[0]) == expected_ports def test_should_add_component_specific_labels(self): docs = render_chart( diff --git a/tests/charts/helm_template_generator.py b/tests/charts/helm_template_generator.py index 2cdcce72ce155..e34f5d5d01411 100644 --- a/tests/charts/helm_template_generator.py +++ b/tests/charts/helm_template_generator.py @@ -131,6 +131,7 @@ def render_chart( kubernetes_version, "--namespace", namespace, + "--debug", ] if show_only: for i in show_only: From 1cc737a53b348092cbb72d1c5ae47f7fcf4115dc Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:13:17 -0700 Subject: [PATCH 06/21] remove debug flag --- tests/charts/helm_template_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/charts/helm_template_generator.py b/tests/charts/helm_template_generator.py index e34f5d5d01411..2cdcce72ce155 100644 --- a/tests/charts/helm_template_generator.py +++ b/tests/charts/helm_template_generator.py @@ -131,7 +131,6 @@ def render_chart( kubernetes_version, "--namespace", namespace, - "--debug", ] if show_only: for i in show_only: From ff7431964fb0a0c9fab8d84bd18488374304d5c7 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:14:54 -0700 Subject: [PATCH 07/21] fix test --- helm_tests/webserver/test_rpc_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm_tests/webserver/test_rpc_server.py b/helm_tests/webserver/test_rpc_server.py index d6b6abd02108d..5db8748413c19 100644 --- a/helm_tests/webserver/test_rpc_server.py +++ b/helm_tests/webserver/test_rpc_server.py @@ -831,7 +831,7 @@ def test_defaults(self): {"port": 80}, ], [ - {"port": 8080}, + {"port": 9080}, {"port": 80}, ], ), From 5ee368511b83e817b86b372f40842e34b35f0d0d Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 10:36:05 -0700 Subject: [PATCH 08/21] fix tests --- .../rpc-server/rpc-server-ingress.yaml | 113 ------------- .../rpc-server/rpc-server-networkpolicy.yaml | 5 +- chart/values.yaml | 46 +----- .../airflow_aux/test_basic_helm_chart.py | 2 +- .../test_rpc_server.py | 148 ++++++++++-------- 5 files changed, 85 insertions(+), 229 deletions(-) delete mode 100644 chart/templates/rpc-server/rpc-server-ingress.yaml rename helm_tests/{webserver => airflow_core}/test_rpc_server.py (91%) diff --git a/chart/templates/rpc-server/rpc-server-ingress.yaml b/chart/templates/rpc-server/rpc-server-ingress.yaml deleted file mode 100644 index 17c2c70297a07..0000000000000 --- a/chart/templates/rpc-server/rpc-server-ingress.yaml +++ /dev/null @@ -1,113 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} - -################################ -## Airflow rpc-server Ingress -################################# -{{- if .Values.rpcServer.enabled }} -{{- if or .Values.ingress.rpcServer.enabled .Values.ingress.enabled }} -{{- $fullname := (include "airflow.fullname" .) }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullname }}-ingress - labels: - tier: airflow - component: airflow-ingress - release: {{ .Release.Name }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: {{ .Release.Service }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} - {{- end }} - {{- with .Values.ingress.rpcServer.annotations }} - annotations: {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.rpcServer.hosts (.Values.ingress.rpcServer.hosts | first | kindIs "string" | not) }} - {{- $anyTlsHosts := false -}} - {{- range .Values.ingress.rpcServer.hosts }} - {{- if .tls }} - {{- if .tls.enabled }} - {{- $anyTlsHosts = true -}} - {{- end }} - {{- end }} - {{- end }} - {{- if $anyTlsHosts }} - tls: - {{- range .Values.ingress.rpcServer.hosts }} - {{- if .tls }} - {{- if .tls.enabled }} - - hosts: - - {{ .name | quote }} - secretName: {{ .tls.secretName }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} - {{- else if .Values.ingress.rpcServer.tls.enabled }} - tls: - - hosts: - {{- .Values.ingress.rpcServer.hosts | default (list .Values.ingress.rpcServer.host) | toYaml | nindent 8 }} - secretName: {{ .Values.ingress.rpcServer.tls.secretName }} - {{- end }} - rules: - {{- range .Values.ingress.rpcServer.hosts | default (list .Values.ingress.rpcServer.host) }} - - http: - paths: - {{- range $.Values.ingress.rpcServer.precedingPaths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ .serviceName }} - port: - name: {{ .servicePort }} - {{- end }} - - backend: - service: - name: {{ $fullname }}-rpc-server - port: - name: airflow-ui - {{- if $.Values.ingress.rpcServer.path }} - path: {{ $.Values.ingress.rpcServer.path }} - pathType: {{ $.Values.ingress.rpcServer.pathType }} - {{- end }} - {{- range $.Values.ingress.rpcServer.succeedingPaths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ .serviceName }} - port: - name: {{ .servicePort }} - {{- end }} - {{- $hostname := . -}} - {{- if . | kindIs "string" | not }} - {{- $hostname = .name -}} - {{- end }} - {{- if $hostname }} - host: {{ tpl $hostname $ | quote }} - {{- end }} - {{- end }} - {{- if .Values.ingress.rpcServer.ingressClassName }} - ingressClassName: {{ .Values.ingress.rpcServer.ingressClassName }} - {{- end }} -{{- end }} -{{- end }} diff --git a/chart/templates/rpc-server/rpc-server-networkpolicy.yaml b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml index 1bcde16b16b87..7dd147da2ff5c 100644 --- a/chart/templates/rpc-server/rpc-server-networkpolicy.yaml +++ b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml @@ -22,7 +22,6 @@ ################################# {{- if .Values.rpcServer.enabled }} {{- if .Values.networkPolicies.enabled }} -{{- $from := or .Values.rpcServer.networkPolicy.ingress.from .Values.rpcServer.extraNetworkPolicies }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -44,9 +43,9 @@ spec: release: {{ .Release.Name }} policyTypes: - Ingress - {{- if $from }} + {{- if .Values.rpcServer.networkPolicy.ingress.from }} ingress: - - from: {{- toYaml $from | nindent 6 }} + - from: {{- toYaml .Values.rpcServer.networkPolicy.ingress.from | nindent 6 }} ports: {{ range .Values.rpcServer.networkPolicy.ingress.ports }} - diff --git a/chart/values.yaml b/chart/values.yaml index 0d2f54654a594..a2942ec2a68fa 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -185,50 +185,6 @@ ingress: # Http paths to add to the web Ingress after the default path succeedingPaths: [] - # Configs for the Ingress of the RPC server Service - rpcServer: - # Enable RPC server ingress resource - enabled: false - - # Annotations for the RPC server Ingress - annotations: {} - - # The path for the RPC server Ingress - path: "/" - - # The pathType for the above path (used only with Kubernetes v1.19 and above) - pathType: "ImplementationSpecific" - - # The hostname for the RPC server Ingress (Deprecated - renamed to `ingress.RPC server.hosts`) - host: "" - - # The hostnames or hosts configuration for the RPC server Ingress - hosts: [] - # # The hostname for the RPC server Ingress (can be templated) - # - name: "" - # # configs for RPC server Ingress TLS - # tls: - # # Enable TLS termination for the RPC server Ingress - # enabled: false - # # the name of a pre-created Secret containing a TLS private key and certificate - # secretName: "" - - # The Ingress Class for the RPC server Ingress (used only with Kubernetes v1.19 and above) - ingressClassName: "" - - # configs for RPC server Ingress TLS (Deprecated - renamed to `ingress.RPC server.hosts[*].tls`) - tls: - # Enable TLS termination for the RPC server Ingress - enabled: false - # the name of a pre-created Secret containing a TLS private key and certificate - secretName: "" - - # HTTP paths to add to the RPC server Ingress before the default path - precedingPaths: [] - - # Http paths to add to the RPC server Ingress after the default path - succeedingPaths: [] - # Configs for the Ingress of the flower Service flower: # Enable web ingress resource @@ -1201,7 +1157,7 @@ migrateDatabaseJob: applyCustomEnv: true rpcServer: - enabled: true + enabled: false baseUrl: "" # Labels specific to workers objects and pods diff --git a/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm_tests/airflow_aux/test_basic_helm_chart.py index d4a32b257b667..5babd79563ea9 100644 --- a/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -27,7 +27,7 @@ from tests.charts.helm_template_generator import render_chart -OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 38 +OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 35 class TestBaseChartTest: diff --git a/helm_tests/webserver/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py similarity index 91% rename from helm_tests/webserver/test_rpc_server.py rename to helm_tests/airflow_core/test_rpc_server.py index 5db8748413c19..448b09eb3e359 100644 --- a/helm_tests/webserver/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -25,22 +25,20 @@ class TestRPCServerDeployment: """Tests rpc-server deployment.""" - def test_can_be_disabled(self): + def test_is_disabled_by_default(self): """ - RPC server should be able to be disabled if the users desires. - - For example, user may be disabled when using rpc-server and having it deployed on another host. + RPC server should be disabled by default. """ docs = render_chart( - values={"rpcServer": {"enabled": False}}, + values={"rpcServer": {}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) - assert 0 == len(docs) + assert len(docs) == 0 def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"rpcServer": {"baseUrl": "https://example.com:21222/mypath/path"}}, + values={"rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -56,7 +54,7 @@ def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(sel def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"rpcServer": {"baseUrl": "https://example.com:21222/mypath/path"}}, + values={"rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -78,7 +76,11 @@ def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): [(8, 10), (10, 8), (8, None), (None, 10), (None, None)], ) def test_revision_history_limit(self, revision_history_limit, global_revision_history_limit): - values = {"rpcServer": {}} + values = { + "rpcServer": { + "enabled": True, + } + } if revision_history_limit: values["rpcServer"]["revisionHistoryLimit"] = revision_history_limit if global_revision_history_limit: @@ -90,9 +92,14 @@ def test_revision_history_limit(self, revision_history_limit, global_revision_hi expected_result = revision_history_limit if revision_history_limit else global_revision_history_limit assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected_result - @pytest.mark.parametrize("values", [{"config": {"rpcServer": {"base_url": ""}}}, {}]) + @pytest.mark.parametrize( + "values", + [ + {"rpcServer": {"enabled": True, "baseUrl": ""}}, + {"rpcServer": {"enabled": True}}, + ], + ) def test_should_not_contain_host_header(self, values): - print(values) docs = render_chart(values=values, show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) assert ( @@ -112,7 +119,8 @@ def test_should_use_templated_base_url_for_probes(self): docs = render_chart( values={ "rpcServer": { - "baseUrl": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path" + "enabled": True, + "baseUrl": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path", }, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -142,6 +150,7 @@ def test_should_add_scheme_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "livenessProbe": {"scheme": "HTTPS"}, "readinessProbe": {"scheme": "HTTPS"}, "startupProbe": {"scheme": "HTTPS"}, @@ -165,6 +174,7 @@ def test_should_add_extra_containers(self): values={ "executor": "CeleryExecutor", "rpcServer": { + "enabled": True, "extraContainers": [ {"name": "{{.Chart.Name}}", "image": "test-registry/test-repo:test-tag"} ], @@ -182,6 +192,7 @@ def test_should_add_extraEnvs(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], }, }, @@ -196,6 +207,7 @@ def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], "extraVolumeMounts": [ {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} @@ -216,6 +228,7 @@ def test_should_add_extra_volume_and_extra_volume_mount(self): def test_should_add_global_volume_and_global_volume_mount(self): docs = render_chart( values={ + "rpcServer": {"enabled": True}, "volumes": [{"name": "test-volume", "emptyDir": {}}], "volumeMounts": [{"name": "test-volume", "mountPath": "/opt/test"}], }, @@ -231,6 +244,7 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "waitForMigrations": { "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], }, @@ -254,6 +268,7 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self): def test_wait_for_migration_airflow_version(self, airflow_version, expected_arg): docs = render_chart( values={ + "rpcServer": {"enabled": True}, "airflowVersion": airflow_version, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -266,6 +281,7 @@ def test_disable_wait_for_migration(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "waitForMigrations": {"enabled": False}, }, }, @@ -280,6 +296,7 @@ def test_should_add_extra_init_containers(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} ], @@ -297,6 +314,7 @@ def test_should_add_component_specific_labels(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "labels": {"test_label": "test_label_value"}, }, }, @@ -310,6 +328,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -351,7 +370,10 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): ) def test_should_create_default_affinity(self): - docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + docs = render_chart( + values={"rpcServer": {"enabled": True}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) assert {"component": "rpc-server"} == jmespath.search( "spec.template.spec.affinity.podAntiAffinity." @@ -384,6 +406,7 @@ def test_affinity_tolerations_topology_spread_constraints_and_node_selector_prec docs = render_chart( values={ "rpcServer": { + "enabled": True, "affinity": expected_affinity, "tolerations": [ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} @@ -435,7 +458,7 @@ def test_affinity_tolerations_topology_spread_constraints_and_node_selector_prec def test_scheduler_name(self): docs = render_chart( - values={"schedulerName": "airflow-scheduler"}, + values={"rpcServer": {"enabled": True}, "schedulerName": "airflow-scheduler"}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -454,7 +477,7 @@ def test_scheduler_name(self): ) def test_logs_persistence_adds_volume_and_mount(self, log_persistence_values, expected_claim_name): docs = render_chart( - values={"logs": {"persistence": log_persistence_values}}, + values={"rpcServer": {"enabled": True}, "logs": {"persistence": log_persistence_values}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -474,7 +497,10 @@ def test_logs_persistence_adds_volume_and_mount(self, log_persistence_values, ex ] def test_config_volumes(self): - docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) + docs = render_chart( + values={"rpcServer": {"enabled": True}}, + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) # default config assert { @@ -488,10 +514,11 @@ def test_rpc_server_resources_are_configurable(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "resources": { "limits": {"cpu": "200m", "memory": "128Mi"}, "requests": {"cpu": "300m", "memory": "169Mi"}, - } + }, }, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -521,6 +548,7 @@ def test_rpc_server_security_contexts_are_configurable(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "securityContexts": { "pod": { "fsGroup": 1000, @@ -532,7 +560,7 @@ def test_rpc_server_security_contexts_are_configurable(self): "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, }, - } + }, }, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -552,12 +580,13 @@ def test_rpc_server_security_context_legacy(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "securityContext": { "fsGroup": 1000, "runAsGroup": 1001, "runAsNonRoot": True, "runAsUser": 2000, - } + }, }, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -572,6 +601,7 @@ def test_rpc_server_security_context_legacy(self): def test_rpc_server_resources_are_not_added_by_default(self): docs = render_chart( + values={"rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == {} @@ -588,7 +618,7 @@ def test_rpc_server_resources_are_not_added_by_default(self): ) def test_default_update_strategy(self, airflow_version, expected_strategy): docs = render_chart( - values={"airflowVersion": airflow_version}, + values={"rpcServer": {"enabled": True}, "airflowVersion": airflow_version}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -597,38 +627,17 @@ def test_default_update_strategy(self, airflow_version, expected_strategy): def test_update_strategy(self): expected_strategy = {"type": "RollingUpdate", "rollingUpdate": {"maxUnavailable": 1}} docs = render_chart( - values={"rpcServer": {"strategy": expected_strategy}}, + values={"rpcServer": {"enabled": True, "strategy": expected_strategy}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) assert jmespath.search("spec.strategy", docs[0]) == expected_strategy - def test_no_airflow_local_settings(self): - docs = render_chart( - values={"airflowLocalSettings": None}, - show_only=["templates/rpc-server/rpc-server-deployment.yaml"], - ) - volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) - assert "airflow_local_settings.py" not in str(volume_mounts) - volume_mounts_init = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) - assert "airflow_local_settings.py" not in str(volume_mounts_init) - - def test_airflow_local_settings(self): + def test_default_command_and_args(self): docs = render_chart( - values={"airflowLocalSettings": "# Well hello!"}, + values={"rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) - volume_mount = { - "name": "config", - "mountPath": "/opt/airflow/config/airflow_local_settings.py", - "subPath": "airflow_local_settings.py", - "readOnly": True, - } - assert volume_mount in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) - assert volume_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0]) - - def test_default_command_and_args(self): - docs = render_chart(show_only=["templates/rpc-server/rpc-server-deployment.yaml"]) assert jmespath.search("spec.template.spec.containers[0].command", docs[0]) == ["bash"] assert ["-c", "exec airflow internal-api"] == jmespath.search( @@ -639,7 +648,7 @@ def test_default_command_and_args(self): @pytest.mark.parametrize("args", [None, ["custom", "args"]]) def test_command_and_args_overrides(self, command, args): docs = render_chart( - values={"rpcServer": {"command": command, "args": args}}, + values={"rpcServer": {"enabled": True, "command": command, "args": args}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -648,7 +657,13 @@ def test_command_and_args_overrides(self, command, args): def test_command_and_args_overrides_are_templated(self): docs = render_chart( - values={"rpcServer": {"command": ["{{ .Release.Name }}"], "args": ["{{ .Release.Service }}"]}}, + values={ + "rpcServer": { + "enabled": True, + "command": ["{{ .Release.Name }}"], + "args": ["{{ .Release.Service }}"], + } + }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -659,6 +674,7 @@ def test_should_add_component_specific_annotations(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "annotations": {"test_annotation": "test_annotation_value"}, }, }, @@ -671,6 +687,7 @@ def test_rpc_server_pod_hostaliases(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "hostAliases": [{"ip": "127.0.0.1", "hostnames": ["foo.local"]}], }, }, @@ -685,7 +702,10 @@ class TestRPCServerService: """Tests rpc-server service.""" def test_default_service(self): - docs = render_chart(show_only=["templates/rpc-server/rpc-server-service.yaml"]) + docs = render_chart( + values={"rpcServer": {"enabled": True}}, + show_only=["templates/rpc-server/rpc-server-service.yaml"], + ) assert "release-name-rpc-server" == jmespath.search("metadata.name", docs[0]) assert jmespath.search("metadata.annotations", docs[0]) is None @@ -700,12 +720,13 @@ def test_overrides(self): values={ "ports": {"rpcServer": 9000}, "rpcServer": { + "enabled": True, "service": { "type": "LoadBalancer", "loadBalancerIP": "127.0.0.1", "annotations": {"foo": "bar"}, "loadBalancerSourceRanges": ["10.123.0.0/16"], - } + }, }, }, show_only=["templates/rpc-server/rpc-server-service.yaml"], @@ -741,7 +762,7 @@ def test_overrides(self): def test_ports_overrides(self, ports, expected_ports): docs = render_chart( values={ - "rpcServer": {"service": {"ports": ports}}, + "rpcServer": {"enabled": True, "service": {"ports": ports}}, }, show_only=["templates/rpc-server/rpc-server-service.yaml"], ) @@ -752,6 +773,7 @@ def test_should_add_component_specific_labels(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "labels": {"test_label": "test_label_value"}, }, }, @@ -777,10 +799,11 @@ def test_nodeport_service(self, ports, expected_ports): docs = render_chart( values={ "rpcServer": { + "enabled": True, "service": { "type": "NodePort", "ports": ports, - } + }, }, }, show_only=["templates/rpc-server/rpc-server-service.yaml"], @@ -804,11 +827,12 @@ def test_defaults(self): values={ "networkPolicies": {"enabled": True}, "rpcServer": { + "enabled": True, "networkPolicy": { "ingress": { "from": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] } - } + }, }, }, show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], @@ -842,12 +866,13 @@ def test_ports_overrides(self, ports, expected_ports): values={ "networkPolicies": {"enabled": True}, "rpcServer": { + "enabled": True, "networkPolicy": { "ingress": { "from": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}], "ports": ports, } - } + }, }, }, show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], @@ -855,26 +880,12 @@ def test_ports_overrides(self, ports, expected_ports): assert expected_ports == jmespath.search("spec.ingress[0].ports", docs[0]) - def test_deprecated_from_param(self): - docs = render_chart( - values={ - "networkPolicies": {"enabled": True}, - "rpcServer": { - "extraNetworkPolicies": [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] - }, - }, - show_only=["templates/rpc-server/rpc-server-networkpolicy.yaml"], - ) - - assert [{"namespaceSelector": {"matchLabels": {"release": "myrelease"}}}] == jmespath.search( - "spec.ingress[0].from", docs[0] - ) - def test_should_add_component_specific_labels(self): docs = render_chart( values={ "networkPolicies": {"enabled": True}, "rpcServer": { + "enabled": True, "labels": {"test_label": "test_label_value"}, }, }, @@ -891,6 +902,7 @@ def test_should_add_component_specific_labels(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "serviceAccount": {"create": True}, "labels": {"test_label": "test_label_value"}, }, @@ -904,6 +916,7 @@ def test_default_automount_service_account_token(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "serviceAccount": {"create": True}, }, }, @@ -915,6 +928,7 @@ def test_overridden_automount_service_account_token(self): docs = render_chart( values={ "rpcServer": { + "enabled": True, "serviceAccount": {"create": True, "automountServiceAccountToken": False}, }, }, From 6237136acdf17b7bd89618e91698ebecbd2b6a6d Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 19:45:57 -0700 Subject: [PATCH 09/21] fix schema --- chart/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 0f78678c39df7..8fbb7bdb9ac4a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -4245,7 +4245,7 @@ "enabled": { "description": "Enable RPC server", "type": "boolean", - "default": true + "default": false }, "baseUrl": { "description": "RPC server base URL", From e6ccbe6f9b6320750cdeea5a4a80efead2a192e5 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 20:03:44 -0700 Subject: [PATCH 10/21] fix helm tests --- .../airflow_aux/test_basic_helm_chart.py | 101 +++++++++--------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm_tests/airflow_aux/test_basic_helm_chart.py index 5babd79563ea9..9ff08849ce394 100644 --- a/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -29,6 +29,45 @@ OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 35 +DEFAULT_OBJECTS_STD_NAMING = { + ("ServiceAccount", "test-basic-airflow-create-user-job"), + ("ServiceAccount", "test-basic-airflow-migrate-database-job"), + ("ServiceAccount", "test-basic-airflow-redis"), + ("ServiceAccount", "test-basic-airflow-scheduler"), + ("ServiceAccount", "test-basic-airflow-statsd"), + ("ServiceAccount", "test-basic-airflow-triggerer"), + ("ServiceAccount", "test-basic-airflow-webserver"), + ("ServiceAccount", "test-basic-airflow-worker"), + ("Secret", "test-basic-airflow-metadata"), + ("Secret", "test-basic-broker-url"), + ("Secret", "test-basic-fernet-key"), + ("Secret", "test-basic-airflow-webserver-secret-key"), + ("Secret", "test-basic-redis-password"), + ("Secret", "test-basic-postgresql"), + ("ConfigMap", "test-basic-airflow-config"), + ("ConfigMap", "test-basic-airflow-statsd"), + ("Role", "test-basic-airflow-pod-launcher-role"), + ("Role", "test-basic-airflow-pod-log-reader-role"), + ("RoleBinding", "test-basic-airflow-pod-launcher-rolebinding"), + ("RoleBinding", "test-basic-airflow-pod-log-reader-rolebinding"), + ("Service", "test-basic-airflow-redis"), + ("Service", "test-basic-airflow-statsd"), + ("Service", "test-basic-airflow-triggerer"), + ("Service", "test-basic-airflow-webserver"), + ("Service", "test-basic-airflow-worker"), + ("Service", "test-basic-postgresql"), + ("Service", "test-basic-postgresql-hl"), + ("Deployment", "test-basic-airflow-scheduler"), + ("Deployment", "test-basic-airflow-statsd"), + ("Deployment", "test-basic-airflow-webserver"), + ("StatefulSet", "test-basic-airflow-redis"), + ("StatefulSet", "test-basic-airflow-worker"), + ("StatefulSet", "test-basic-airflow-triggerer"), + ("StatefulSet", "test-basic-postgresql"), + ("Job", "test-basic-airflow-create-user"), + ("Job", "test-basic-airflow-run-airflow-migrations"), +} + class TestBaseChartTest: """Tests basic helm chart tests.""" @@ -66,7 +105,6 @@ def test_basic_deployments(self, version): ("ServiceAccount", "test-basic-create-user-job"), ("ServiceAccount", "test-basic-migrate-database-job"), ("ServiceAccount", "test-basic-redis"), - ("ServiceAccount", "test-basic-rpc-server"), ("ServiceAccount", "test-basic-scheduler"), ("ServiceAccount", "test-basic-statsd"), ("ServiceAccount", "test-basic-triggerer"), @@ -87,13 +125,11 @@ def test_basic_deployments(self, version): ("Service", "test-basic-postgresql-hl"), ("Service", "test-basic-postgresql"), ("Service", "test-basic-redis"), - ("Service", "test-basic-rpc-server"), ("Service", "test-basic-statsd"), ("Service", "test-basic-webserver"), ("Service", "test-basic-worker"), ("Deployment", "test-basic-scheduler"), ("Deployment", "test-basic-statsd"), - ("Deployment", "test-basic-rpc-server"), (self.default_trigger_obj(version), "test-basic-triggerer"), ("Deployment", "test-basic-webserver"), ("StatefulSet", "test-basic-postgresql"), @@ -126,51 +162,21 @@ def test_basic_deployments_with_standard_naming(self): "test-basic", {"useStandardNaming": True}, ) - list_of_kind_names_tuples = { - (k8s_object["kind"], k8s_object["metadata"]["name"]) for k8s_object in k8s_objects - } - expected = { - ("ServiceAccount", "test-basic-airflow-create-user-job"), - ("ServiceAccount", "test-basic-airflow-migrate-database-job"), - ("ServiceAccount", "test-basic-airflow-redis"), - ("ServiceAccount", "test-basic-airflow-scheduler"), - ("ServiceAccount", "test-basic-airflow-statsd"), - ("ServiceAccount", "test-basic-airflow-triggerer"), - ("ServiceAccount", "test-basic-airflow-webserver"), - ("ServiceAccount", "test-basic-airflow-rpc-server"), - ("ServiceAccount", "test-basic-airflow-worker"), - ("Secret", "test-basic-airflow-metadata"), - ("Secret", "test-basic-broker-url"), - ("Secret", "test-basic-fernet-key"), - ("Secret", "test-basic-airflow-webserver-secret-key"), - ("Secret", "test-basic-redis-password"), - ("Secret", "test-basic-postgresql"), - ("ConfigMap", "test-basic-airflow-config"), - ("ConfigMap", "test-basic-airflow-statsd"), - ("Role", "test-basic-airflow-pod-launcher-role"), - ("Role", "test-basic-airflow-pod-log-reader-role"), - ("RoleBinding", "test-basic-airflow-pod-launcher-rolebinding"), - ("RoleBinding", "test-basic-airflow-pod-log-reader-rolebinding"), - ("Service", "test-basic-airflow-redis"), - ("Service", "test-basic-airflow-statsd"), - ("Service", "test-basic-airflow-triggerer"), - ("Service", "test-basic-airflow-webserver"), - ("Service", "test-basic-airflow-rpc-server"), - ("Service", "test-basic-airflow-worker"), - ("Service", "test-basic-postgresql"), - ("Service", "test-basic-postgresql-hl"), - ("Deployment", "test-basic-airflow-scheduler"), - ("Deployment", "test-basic-airflow-statsd"), - ("Deployment", "test-basic-airflow-webserver"), + actual = {(x["kind"], x["metadata"]["name"]) for x in k8s_objects} + assert actual == DEFAULT_OBJECTS_STD_NAMING + + def test_basic_deployment_with_rpc_server(self): + extra_objects = { ("Deployment", "test-basic-airflow-rpc-server"), - ("StatefulSet", "test-basic-airflow-redis"), - ("StatefulSet", "test-basic-airflow-worker"), - ("StatefulSet", "test-basic-airflow-triggerer"), - ("StatefulSet", "test-basic-postgresql"), - ("Job", "test-basic-airflow-create-user"), - ("Job", "test-basic-airflow-run-airflow-migrations"), + ("Service", "test-basic-airflow-rpc-server"), + ("ServiceAccount", "test-basic-airflow-rpc-server"), } - assert list_of_kind_names_tuples == expected + k8s_objects = render_chart( + "test-basic", + values={"rpcServer": {"enabled": True}, "useStandardNaming": True}, + ) + actual = {(x["kind"], x["metadata"]["name"]) for x in k8s_objects} + assert actual == (DEFAULT_OBJECTS_STD_NAMING | extra_objects) @pytest.mark.parametrize("version", ["2.3.2", "2.4.0", "default"]) def test_basic_deployment_with_standalone_dag_processor(self, version): @@ -202,7 +208,6 @@ def test_basic_deployment_with_standalone_dag_processor(self, version): ("ServiceAccount", "test-basic-statsd"), ("ServiceAccount", "test-basic-triggerer"), ("ServiceAccount", "test-basic-dag-processor"), - ("ServiceAccount", "test-basic-rpc-server"), ("ServiceAccount", "test-basic-webserver"), ("ServiceAccount", "test-basic-worker"), ("Secret", "test-basic-metadata"), @@ -222,14 +227,12 @@ def test_basic_deployment_with_standalone_dag_processor(self, version): ("Service", "test-basic-redis"), ("Service", "test-basic-statsd"), ("Service", "test-basic-webserver"), - ("Service", "test-basic-rpc-server"), ("Service", "test-basic-worker"), ("Deployment", "test-basic-scheduler"), ("Deployment", "test-basic-statsd"), (self.default_trigger_obj(version), "test-basic-triggerer"), ("Deployment", "test-basic-dag-processor"), ("Deployment", "test-basic-webserver"), - ("Deployment", "test-basic-rpc-server"), ("StatefulSet", "test-basic-postgresql"), ("StatefulSet", "test-basic-redis"), ("StatefulSet", "test-basic-worker"), From 9e0973183be129b53abdc604930cfd6249ed4593 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 20:08:22 -0700 Subject: [PATCH 11/21] fix more tests --- helm_tests/security/test_rbac.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/helm_tests/security/test_rbac.py b/helm_tests/security/test_rbac.py index fff2883f28263..ae23a5af63f88 100644 --- a/helm_tests/security/test_rbac.py +++ b/helm_tests/security/test_rbac.py @@ -141,7 +141,7 @@ def test_deployments_no_rbac_no_sa(self, version): "redis": {"serviceAccount": {"create": False}}, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, - "rpcServer": {"serviceAccount": {"create": False}}, + "rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "statsd": {"serviceAccount": {"create": False}}, @@ -194,7 +194,7 @@ def test_deployments_with_rbac_no_sa(self, version): }, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, - "rpcServer": {"serviceAccount": {"create": False}}, + "rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "flower": {"enabled": True, "serviceAccount": {"create": False}}, @@ -228,6 +228,7 @@ def test_deployments_with_rbac_with_sa(self, version): "cleanup": {"enabled": True}, "flower": {"enabled": True}, "pgbouncer": {"enabled": True}, + "rpcServer": {"enabled": True}, }, version=version, ), @@ -253,7 +254,7 @@ def test_service_account_custom_names(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, - "rpcServer": {"serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, + "rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, @@ -290,7 +291,7 @@ def test_service_account_custom_names_in_objects(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, - "rpcServer": {"serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, + "rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, From 0e2c74d7b8f9fbc22b96e752459394a286e9a107 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 20:32:33 -0700 Subject: [PATCH 12/21] rename rpcServer to _rpcServer and add newsfragment / docs re experimental / dev only --- chart/templates/_helpers.yaml | 6 +- .../rpc-server/rpc-server-deployment.yaml | 140 +++++++++--------- .../rpc-server/rpc-server-networkpolicy.yaml | 12 +- .../rpc-server-poddisruptionbudget.yaml | 10 +- .../rpc-server/rpc-server-service.yaml | 20 +-- .../rpc-server/rpc-server-serviceaccount.yaml | 10 +- chart/values.schema.json | 12 +- chart/values.yaml | 10 +- .../airflow_aux/test_basic_helm_chart.py | 2 +- helm_tests/airflow_core/test_rpc_server.py | 104 ++++++------- helm_tests/security/test_rbac.py | 14 +- 11 files changed, 173 insertions(+), 167 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index f834fbc5a1309..aaa86ecbe88b2 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -585,10 +585,10 @@ server_tls_key_file = /etc/pgbouncer/server.key {{/* Create the name of the RPC server service account to use */}} {{- define "rpcServer.serviceAccountName" -}} - {{- if .Values.rpcServer.serviceAccount.create }} - {{- default (printf "%s-rpc-server" (include "airflow.serviceAccountName" .)) .Values.rpcServer.serviceAccount.name }} + {{- if .Values._rpcServer.serviceAccount.create }} + {{- default (printf "%s-rpc-server" (include "airflow.serviceAccountName" .)) .Values._rpcServer.serviceAccount.name }} {{- else }} - {{- default "default" .Values.rpcServer.serviceAccount.name }} + {{- default "default" .Values._rpcServer.serviceAccount.name }} {{- end }} {{- end }} diff --git a/chart/templates/rpc-server/rpc-server-deployment.yaml b/chart/templates/rpc-server/rpc-server-deployment.yaml index fb900b4963112..f0fe851f8059a 100644 --- a/chart/templates/rpc-server/rpc-server-deployment.yaml +++ b/chart/templates/rpc-server/rpc-server-deployment.yaml @@ -20,16 +20,16 @@ ################################ ## Airflow rpc-server Deployment ################################# -{{- if .Values.rpcServer.enabled }} -{{- $nodeSelector := or .Values.rpcServer.nodeSelector .Values.nodeSelector }} -{{- $affinity := or .Values.rpcServer.affinity .Values.affinity }} -{{- $tolerations := or .Values.rpcServer.tolerations .Values.tolerations }} -{{- $topologySpreadConstraints := or .Values.rpcServer.topologySpreadConstraints .Values.topologySpreadConstraints }} -{{- $revisionHistoryLimit := or .Values.rpcServer.revisionHistoryLimit .Values.revisionHistoryLimit }} -{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.rpcServer) }} -{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.rpcServer) }} -{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.rpcServer.waitForMigrations) }} -{{- $containerLifecycleHooks := or .Values.rpcServer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- if .Values._rpcServer.enabled }} +{{- $nodeSelector := or .Values._rpcServer.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values._rpcServer.affinity .Values.affinity }} +{{- $tolerations := or .Values._rpcServer.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values._rpcServer.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $revisionHistoryLimit := or .Values._rpcServer.revisionHistoryLimit .Values.revisionHistoryLimit }} +{{- $securityContext := include "airflowPodSecurityContext" (list . .Values._rpcServer) }} +{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values._rpcServer) }} +{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values._rpcServer.waitForMigrations) }} +{{- $containerLifecycleHooks := or .Values._rpcServer.containerLifecycleHooks .Values.containerLifecycleHooks }} apiVersion: apps/v1 kind: Deployment metadata: @@ -43,17 +43,17 @@ metadata: {{- with .Values.labels }} {{- toYaml . | nindent 4 }} {{- end }} - {{- if .Values.rpcServer.annotations }} - annotations: {{- toYaml .Values.rpcServer.annotations | nindent 4 }} + {{- if .Values._rpcServer.annotations }} + annotations: {{- toYaml .Values._rpcServer.annotations | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.rpcServer.replicas }} + replicas: {{ .Values._rpcServer.replicas }} {{- if $revisionHistoryLimit }} revisionHistoryLimit: {{ $revisionHistoryLimit }} {{- end }} strategy: - {{- if .Values.rpcServer.strategy }} - {{- toYaml .Values.rpcServer.strategy | nindent 4 }} + {{- if .Values._rpcServer.strategy }} + {{- toYaml .Values._rpcServer.strategy | nindent 4 }} {{- else }} {{- if semverCompare ">=2.0.0" .Values.airflowVersion }} # Here we define the rolling update strategy @@ -82,8 +82,8 @@ spec: tier: airflow component: rpc-server release: {{ .Release.Name }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 8 }} + {{- if or (.Values.labels) (.Values._rpcServer.labels) }} + {{- mustMerge .Values._rpcServer.labels .Values.labels | toYaml | nindent 8 }} {{- end }} annotations: checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} @@ -94,16 +94,16 @@ spec: {{- if .Values.airflowPodAnnotations }} {{- toYaml .Values.airflowPodAnnotations | nindent 8 }} {{- end }} - {{- if .Values.rpcServer.podAnnotations }} - {{- toYaml .Values.rpcServer.podAnnotations | nindent 8 }} + {{- if .Values._rpcServer.podAnnotations }} + {{- toYaml .Values._rpcServer.podAnnotations | nindent 8 }} {{- end }} spec: - {{- if .Values.rpcServer.hostAliases }} - hostAliases: {{- toYaml .Values.rpcServer.hostAliases | nindent 8 }} + {{- if .Values._rpcServer.hostAliases }} + hostAliases: {{- toYaml .Values._rpcServer.hostAliases | nindent 8 }} {{- end }} serviceAccountName: {{ include "rpcServer.serviceAccountName" . }} - {{- if .Values.rpcServer.priorityClassName }} - priorityClassName: {{ .Values.rpcServer.priorityClassName }} + {{- if .Values._rpcServer.priorityClassName }} + priorityClassName: {{ .Values._rpcServer.priorityClassName }} {{- end }} {{- if .Values.schedulerName }} schedulerName: {{ .Values.schedulerName }} @@ -131,9 +131,9 @@ spec: - name: {{ template "registry_secret" . }} {{- end }} initContainers: - {{- if .Values.rpcServer.waitForMigrations.enabled }} + {{- if .Values._rpcServer.waitForMigrations.enabled }} - name: wait-for-airflow-migrations - resources: {{- toYaml .Values.rpcServer.resources | nindent 12 }} + resources: {{- toYaml .Values._rpcServer.resources | nindent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }} @@ -142,20 +142,20 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.rpcServer.extraVolumeMounts }} - {{- tpl (toYaml .Values.rpcServer.extraVolumeMounts) . | nindent 12 }} + {{- if .Values._rpcServer.extraVolumeMounts }} + {{- tpl (toYaml .Values._rpcServer.extraVolumeMounts) . | nindent 12 }} {{- end }} args: {{- include "wait-for-migrations-command" . | indent 10 }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} - {{- if .Values.rpcServer.waitForMigrations.env }} - {{- tpl (toYaml .Values.rpcServer.waitForMigrations.env) $ | nindent 12 }} + {{- if .Values._rpcServer.waitForMigrations.env }} + {{- tpl (toYaml .Values._rpcServer.waitForMigrations.env) $ | nindent 12 }} {{- end }} {{- end }} - {{- if .Values.rpcServer.extraInitContainers }} - {{- toYaml .Values.rpcServer.extraInitContainers | nindent 8 }} + {{- if .Values._rpcServer.extraInitContainers }} + {{- toYaml .Values._rpcServer.extraInitContainers | nindent 8 }} {{- end }} containers: - name: rpc-server @@ -165,13 +165,13 @@ spec: {{- if $containerLifecycleHooks }} lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }} {{- end }} - {{- if .Values.rpcServer.command }} - command: {{ tpl (toYaml .Values.rpcServer.command) . | nindent 12 }} + {{- if .Values._rpcServer.command }} + command: {{ tpl (toYaml .Values._rpcServer.command) . | nindent 12 }} {{- end }} - {{- if .Values.rpcServer.args }} - args: {{- tpl (toYaml .Values.rpcServer.args) . | nindent 12 }} + {{- if .Values._rpcServer.args }} + args: {{- tpl (toYaml .Values._rpcServer.args) . | nindent 12 }} {{- end }} - resources: {{- toYaml .Values.rpcServer.resources | nindent 12 }} + resources: {{- toYaml .Values._rpcServer.resources | nindent 12 }} volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.logs.persistence.enabled }} @@ -181,63 +181,63 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} - {{- if .Values.rpcServer.extraVolumeMounts }} - {{- tpl (toYaml .Values.rpcServer.extraVolumeMounts) . | nindent 12 }} + {{- if .Values._rpcServer.extraVolumeMounts }} + {{- tpl (toYaml .Values._rpcServer.extraVolumeMounts) . | nindent 12 }} {{- end }} ports: - name: rpc-server - containerPort: {{ .Values.ports.rpcServer }} + containerPort: {{ .Values.ports._rpcServer }} livenessProbe: httpGet: - path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health - port: {{ .Values.ports.rpcServer }} - {{- if .Values.rpcServer.baseUrl}} + path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports._rpcServer }} + {{- if .Values._rpcServer.baseUrl}} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} {{- end }} - scheme: {{ .Values.rpcServer.livenessProbe.scheme | default "http" }} - initialDelaySeconds: {{ .Values.rpcServer.livenessProbe.initialDelaySeconds }} - timeoutSeconds: {{ .Values.rpcServer.livenessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.rpcServer.livenessProbe.failureThreshold }} - periodSeconds: {{ .Values.rpcServer.livenessProbe.periodSeconds }} + scheme: {{ .Values._rpcServer.livenessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values._rpcServer.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values._rpcServer.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values._rpcServer.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values._rpcServer.livenessProbe.periodSeconds }} readinessProbe: httpGet: - path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health - port: {{ .Values.ports.rpcServer }} - {{- if .Values.rpcServer.baseUrl }} + path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports._rpcServer }} + {{- if .Values._rpcServer.baseUrl }} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} {{- end }} - scheme: {{ .Values.rpcServer.readinessProbe.scheme | default "http" }} - initialDelaySeconds: {{ .Values.rpcServer.readinessProbe.initialDelaySeconds }} - timeoutSeconds: {{ .Values.rpcServer.readinessProbe.timeoutSeconds }} - failureThreshold: {{ .Values.rpcServer.readinessProbe.failureThreshold }} - periodSeconds: {{ .Values.rpcServer.readinessProbe.periodSeconds }} + scheme: {{ .Values._rpcServer.readinessProbe.scheme | default "http" }} + initialDelaySeconds: {{ .Values._rpcServer.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values._rpcServer.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values._rpcServer.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values._rpcServer.readinessProbe.periodSeconds }} startupProbe: httpGet: - path: {{ if .Values.rpcServer.baseUrl }}{{- with urlParse (tpl .Values.rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health - port: {{ .Values.ports.rpcServer }} - {{- if .Values.rpcServer.baseUrl}} + path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + port: {{ .Values.ports._rpcServer }} + {{- if .Values._rpcServer.baseUrl}} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} {{- end }} - scheme: {{ .Values.rpcServer.startupProbe.scheme | default "http" }} - timeoutSeconds: {{ .Values.rpcServer.startupProbe.timeoutSeconds }} - failureThreshold: {{ .Values.rpcServer.startupProbe.failureThreshold }} - periodSeconds: {{ .Values.rpcServer.startupProbe.periodSeconds }} + scheme: {{ .Values._rpcServer.startupProbe.scheme | default "http" }} + timeoutSeconds: {{ .Values._rpcServer.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values._rpcServer.startupProbe.failureThreshold }} + periodSeconds: {{ .Values._rpcServer.startupProbe.periodSeconds }} envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} - {{- include "container_extra_envs" (list . .Values.rpcServer.env) | indent 10 }} + {{- include "container_extra_envs" (list . .Values._rpcServer.env) | indent 10 }} {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }} {{- include "git_sync_container" . | nindent 8 }} {{- end }} - {{- if .Values.rpcServer.extraContainers }} - {{- tpl (toYaml .Values.rpcServer.extraContainers) . | nindent 8 }} + {{- if .Values._rpcServer.extraContainers }} + {{- tpl (toYaml .Values._rpcServer.extraContainers) . | nindent 8 }} {{- end }} volumes: - name: config @@ -253,7 +253,7 @@ spec: {{- if .Values.volumes }} {{- toYaml .Values.volumes | nindent 8 }} {{- end }} - {{- if .Values.rpcServer.extraVolumes }} - {{- tpl (toYaml .Values.rpcServer.extraVolumes) . | nindent 8 }} + {{- if .Values._rpcServer.extraVolumes }} + {{- tpl (toYaml .Values._rpcServer.extraVolumes) . | nindent 8 }} {{- end }} {{- end }} diff --git a/chart/templates/rpc-server/rpc-server-networkpolicy.yaml b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml index 7dd147da2ff5c..ce3453429450b 100644 --- a/chart/templates/rpc-server/rpc-server-networkpolicy.yaml +++ b/chart/templates/rpc-server/rpc-server-networkpolicy.yaml @@ -20,7 +20,7 @@ ################################ ## Airflow rpc-server NetworkPolicy ################################# -{{- if .Values.rpcServer.enabled }} +{{- if .Values._rpcServer.enabled }} {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -32,8 +32,8 @@ metadata: release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- if or (.Values.labels) (.Values._rpcServer.labels) }} + {{- mustMerge .Values._rpcServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: podSelector: @@ -43,11 +43,11 @@ spec: release: {{ .Release.Name }} policyTypes: - Ingress - {{- if .Values.rpcServer.networkPolicy.ingress.from }} + {{- if .Values._rpcServer.networkPolicy.ingress.from }} ingress: - - from: {{- toYaml .Values.rpcServer.networkPolicy.ingress.from | nindent 6 }} + - from: {{- toYaml .Values._rpcServer.networkPolicy.ingress.from | nindent 6 }} ports: - {{ range .Values.rpcServer.networkPolicy.ingress.ports }} + {{ range .Values._rpcServer.networkPolicy.ingress.ports }} - {{- range $key, $val := . }} {{ $key }}: {{ tpl (toString $val) $ }} diff --git a/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml b/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml index bf722b89542fd..e6d4afebb2da0 100644 --- a/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml +++ b/chart/templates/rpc-server/rpc-server-poddisruptionbudget.yaml @@ -20,8 +20,8 @@ ################################ ## Airflow rpc-server PodDisruptionBudget ################################# -{{- if .Values.rpcServer.enabled }} -{{- if .Values.rpcServer.podDisruptionBudget.enabled }} +{{- if .Values._rpcServer.enabled }} +{{- if .Values._rpcServer.podDisruptionBudget.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -32,8 +32,8 @@ metadata: release: {{ .Release.Name }} chart: {{ .Chart.Name }} heritage: {{ .Release.Service }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- if or (.Values.labels) (.Values._rpcServer.labels) }} + {{- mustMerge .Values._rpcServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} spec: selector: @@ -41,6 +41,6 @@ spec: tier: airflow component: rpc-server release: {{ .Release.Name }} - {{- toYaml .Values.rpcServer.podDisruptionBudget.config | nindent 2 }} + {{- toYaml .Values._rpcServer.podDisruptionBudget.config | nindent 2 }} {{- end }} {{- end }} diff --git a/chart/templates/rpc-server/rpc-server-service.yaml b/chart/templates/rpc-server/rpc-server-service.yaml index 68da246b0a7fd..f6b9160d2fb48 100644 --- a/chart/templates/rpc-server/rpc-server-service.yaml +++ b/chart/templates/rpc-server/rpc-server-service.yaml @@ -20,7 +20,7 @@ ################################ ## Airflow rpc-server Service ################################# -{{- if .Values.rpcServer.enabled }} +{{- if .Values._rpcServer.enabled }} apiVersion: v1 kind: Service metadata: @@ -31,29 +31,29 @@ metadata: release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- if or (.Values.labels) (.Values._rpcServer.labels) }} + {{- mustMerge .Values._rpcServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} - {{- with .Values.rpcServer.service.annotations }} + {{- with .Values._rpcServer.service.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: - type: {{ .Values.rpcServer.service.type }} + type: {{ .Values._rpcServer.service.type }} selector: tier: airflow component: rpc-server release: {{ .Release.Name }} ports: - {{ range .Values.rpcServer.service.ports }} + {{ range .Values._rpcServer.service.ports }} - {{- range $key, $val := . }} {{ $key }}: {{ tpl (toString $val) $ }} {{- end }} {{- end }} - {{- if .Values.rpcServer.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.rpcServer.service.loadBalancerIP }} + {{- if .Values._rpcServer.service.loadBalancerIP }} + loadBalancerIP: {{ .Values._rpcServer.service.loadBalancerIP }} {{- end }} - {{- if .Values.rpcServer.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: {{- toYaml .Values.rpcServer.service.loadBalancerSourceRanges | nindent 4 }} + {{- if .Values._rpcServer.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values._rpcServer.service.loadBalancerSourceRanges | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/rpc-server/rpc-server-serviceaccount.yaml b/chart/templates/rpc-server/rpc-server-serviceaccount.yaml index 490311e091f43..655b346fbe0ab 100644 --- a/chart/templates/rpc-server/rpc-server-serviceaccount.yaml +++ b/chart/templates/rpc-server/rpc-server-serviceaccount.yaml @@ -20,10 +20,10 @@ ###################################### ## Airflow rpc-server ServiceAccount ###################################### -{{- if and .Values.rpcServer.enabled .Values.rpcServer.serviceAccount.create }} +{{- if and .Values._rpcServer.enabled .Values._rpcServer.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount -automountServiceAccountToken: {{ .Values.rpcServer.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values._rpcServer.serviceAccount.automountServiceAccountToken }} metadata: name: {{ include "rpcServer.serviceAccountName" . }} labels: @@ -32,10 +32,10 @@ metadata: release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} - {{- if or (.Values.labels) (.Values.rpcServer.labels) }} - {{- mustMerge .Values.rpcServer.labels .Values.labels | toYaml | nindent 4 }} + {{- if or (.Values.labels) (.Values._rpcServer.labels) }} + {{- mustMerge .Values._rpcServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} - {{- with .Values.rpcServer.serviceAccount.annotations }} + {{- with .Values._rpcServer.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 8fbb7bdb9ac4a..eb1d4ad63977f 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -4236,8 +4236,8 @@ } } }, - "rpcServer": { - "description": "Airflow RPC server settings.", + "_rpcServer": { + "description": "Airflow RPC server settings (AIP-44). Experimental / for dev purpose only.", "type": "object", "x-docsSection": "RPC Server", "additionalProperties": false, @@ -4529,7 +4529,7 @@ }, "default": [ { - "port": "{{ .Values.ports.rpcServer }}" + "port": "{{ .Values.ports._rpcServer }}" } ], "examples": [ @@ -4793,7 +4793,7 @@ "default": [ { "name": "rpc-server", - "port": "{{ .Values.ports.rpcServer }}" + "port": "{{ .Values.ports._rpcServer }}" } ], "examples": [ @@ -7499,8 +7499,8 @@ "type": "integer", "default": 8080 }, - "rpcServer": { - "description": "RPC server port.", + "_rpcServer": { + "description": "RPC server port (AIP-44). Experimental / dev purpose only.", "type": "integer", "default": 9080 }, diff --git a/chart/values.yaml b/chart/values.yaml index a2942ec2a68fa..b8e0e951ce4b3 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1156,7 +1156,8 @@ migrateDatabaseJob: useHelmHooks: true applyCustomEnv: true -rpcServer: +# rpcServer support is experimental / dev purpose only and will later be renamed +_rpcServer: enabled: false baseUrl: "" @@ -1187,7 +1188,7 @@ rpcServer: annotations: {} ports: - name: rpc-server - port: "{{ .Values.ports.rpcServer }}" + port: "{{ .Values.ports._rpcServer }}" loadBalancerIP: ~ ## Limit load balancer source ips to list of CIDRs @@ -1228,7 +1229,7 @@ rpcServer: from: [] # Ports for webserver NetworkPolicy ingress (if `from` is set) ports: - - port: "{{ .Values.ports.rpcServer }}" + - port: "{{ .Values.ports._rpcServer }}" resources: {} # limits: @@ -2335,7 +2336,6 @@ elasticsearch: ports: flowerUI: 5555 airflowUI: 8080 - rpcServer: 9080 workerLogs: 8793 triggererLogs: 8794 redisDB: 6379 @@ -2343,6 +2343,8 @@ ports: statsdScrape: 9102 pgbouncer: 6543 pgbouncerScrape: 9127 + # rpcServer support is experimental / dev purpose only and will later be renamed + _rpcServer: 9080 # Define any ResourceQuotas for namespace quotas: {} diff --git a/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm_tests/airflow_aux/test_basic_helm_chart.py index 9ff08849ce394..771bae22ed7a9 100644 --- a/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -173,7 +173,7 @@ def test_basic_deployment_with_rpc_server(self): } k8s_objects = render_chart( "test-basic", - values={"rpcServer": {"enabled": True}, "useStandardNaming": True}, + values={"_rpcServer": {"enabled": True}, "useStandardNaming": True}, ) actual = {(x["kind"], x["metadata"]["name"]) for x in k8s_objects} assert actual == (DEFAULT_OBJECTS_STD_NAMING | extra_objects) diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index 448b09eb3e359..9bfb3c8e21499 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -30,7 +30,7 @@ def test_is_disabled_by_default(self): RPC server should be disabled by default. """ docs = render_chart( - values={"rpcServer": {}}, + values={"_rpcServer": {}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -38,7 +38,7 @@ def test_is_disabled_by_default(self): def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, + values={"_rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -54,7 +54,7 @@ def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(sel def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, + values={"_rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -77,12 +77,12 @@ def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): ) def test_revision_history_limit(self, revision_history_limit, global_revision_history_limit): values = { - "rpcServer": { + "_rpcServer": { "enabled": True, } } if revision_history_limit: - values["rpcServer"]["revisionHistoryLimit"] = revision_history_limit + values["_rpcServer"]["revisionHistoryLimit"] = revision_history_limit if global_revision_history_limit: values["revisionHistoryLimit"] = global_revision_history_limit docs = render_chart( @@ -95,8 +95,8 @@ def test_revision_history_limit(self, revision_history_limit, global_revision_hi @pytest.mark.parametrize( "values", [ - {"rpcServer": {"enabled": True, "baseUrl": ""}}, - {"rpcServer": {"enabled": True}}, + {"_rpcServer": {"enabled": True, "baseUrl": ""}}, + {"_rpcServer": {"enabled": True}}, ], ) def test_should_not_contain_host_header(self, values): @@ -118,7 +118,7 @@ def test_should_not_contain_host_header(self, values): def test_should_use_templated_base_url_for_probes(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "baseUrl": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path", }, @@ -149,7 +149,7 @@ def test_should_use_templated_base_url_for_probes(self): def test_should_add_scheme_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "livenessProbe": {"scheme": "HTTPS"}, "readinessProbe": {"scheme": "HTTPS"}, @@ -173,7 +173,7 @@ def test_should_add_extra_containers(self): docs = render_chart( values={ "executor": "CeleryExecutor", - "rpcServer": { + "_rpcServer": { "enabled": True, "extraContainers": [ {"name": "{{.Chart.Name}}", "image": "test-registry/test-repo:test-tag"} @@ -191,7 +191,7 @@ def test_should_add_extra_containers(self): def test_should_add_extraEnvs(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], }, @@ -206,7 +206,7 @@ def test_should_add_extraEnvs(self): def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], "extraVolumeMounts": [ @@ -228,7 +228,7 @@ def test_should_add_extra_volume_and_extra_volume_mount(self): def test_should_add_global_volume_and_global_volume_mount(self): docs = render_chart( values={ - "rpcServer": {"enabled": True}, + "_rpcServer": {"enabled": True}, "volumes": [{"name": "test-volume", "emptyDir": {}}], "volumeMounts": [{"name": "test-volume", "mountPath": "/opt/test"}], }, @@ -243,7 +243,7 @@ def test_should_add_global_volume_and_global_volume_mount(self): def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "waitForMigrations": { "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], @@ -268,7 +268,7 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self): def test_wait_for_migration_airflow_version(self, airflow_version, expected_arg): docs = render_chart( values={ - "rpcServer": {"enabled": True}, + "_rpcServer": {"enabled": True}, "airflowVersion": airflow_version, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], @@ -280,7 +280,7 @@ def test_wait_for_migration_airflow_version(self, airflow_version, expected_arg) def test_disable_wait_for_migration(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "waitForMigrations": {"enabled": False}, }, @@ -295,7 +295,7 @@ def test_disable_wait_for_migration(self): def test_should_add_extra_init_containers(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} @@ -313,7 +313,7 @@ def test_should_add_extra_init_containers(self): def test_should_add_component_specific_labels(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "labels": {"test_label": "test_label_value"}, }, @@ -327,7 +327,7 @@ def test_should_add_component_specific_labels(self): def test_should_create_valid_affinity_tolerations_and_node_selector(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "affinity": { "nodeAffinity": { @@ -371,7 +371,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): def test_should_create_default_affinity(self): docs = render_chart( - values={"rpcServer": {"enabled": True}}, + values={"_rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -405,7 +405,7 @@ def test_affinity_tolerations_topology_spread_constraints_and_node_selector_prec } docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "affinity": expected_affinity, "tolerations": [ @@ -458,7 +458,7 @@ def test_affinity_tolerations_topology_spread_constraints_and_node_selector_prec def test_scheduler_name(self): docs = render_chart( - values={"rpcServer": {"enabled": True}, "schedulerName": "airflow-scheduler"}, + values={"_rpcServer": {"enabled": True}, "schedulerName": "airflow-scheduler"}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -477,7 +477,7 @@ def test_scheduler_name(self): ) def test_logs_persistence_adds_volume_and_mount(self, log_persistence_values, expected_claim_name): docs = render_chart( - values={"rpcServer": {"enabled": True}, "logs": {"persistence": log_persistence_values}}, + values={"_rpcServer": {"enabled": True}, "logs": {"persistence": log_persistence_values}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -498,7 +498,7 @@ def test_logs_persistence_adds_volume_and_mount(self, log_persistence_values, ex def test_config_volumes(self): docs = render_chart( - values={"rpcServer": {"enabled": True}}, + values={"_rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -513,7 +513,7 @@ def test_config_volumes(self): def test_rpc_server_resources_are_configurable(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "resources": { "limits": {"cpu": "200m", "memory": "128Mi"}, @@ -547,7 +547,7 @@ def test_rpc_server_resources_are_configurable(self): def test_rpc_server_security_contexts_are_configurable(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "securityContexts": { "pod": { @@ -579,7 +579,7 @@ def test_rpc_server_security_contexts_are_configurable(self): def test_rpc_server_security_context_legacy(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "securityContext": { "fsGroup": 1000, @@ -601,7 +601,7 @@ def test_rpc_server_security_context_legacy(self): def test_rpc_server_resources_are_not_added_by_default(self): docs = render_chart( - values={"rpcServer": {"enabled": True}}, + values={"_rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == {} @@ -618,7 +618,7 @@ def test_rpc_server_resources_are_not_added_by_default(self): ) def test_default_update_strategy(self, airflow_version, expected_strategy): docs = render_chart( - values={"rpcServer": {"enabled": True}, "airflowVersion": airflow_version}, + values={"_rpcServer": {"enabled": True}, "airflowVersion": airflow_version}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -627,7 +627,7 @@ def test_default_update_strategy(self, airflow_version, expected_strategy): def test_update_strategy(self): expected_strategy = {"type": "RollingUpdate", "rollingUpdate": {"maxUnavailable": 1}} docs = render_chart( - values={"rpcServer": {"enabled": True, "strategy": expected_strategy}}, + values={"_rpcServer": {"enabled": True, "strategy": expected_strategy}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -635,7 +635,7 @@ def test_update_strategy(self): def test_default_command_and_args(self): docs = render_chart( - values={"rpcServer": {"enabled": True}}, + values={"_rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -648,7 +648,7 @@ def test_default_command_and_args(self): @pytest.mark.parametrize("args", [None, ["custom", "args"]]) def test_command_and_args_overrides(self, command, args): docs = render_chart( - values={"rpcServer": {"enabled": True, "command": command, "args": args}}, + values={"_rpcServer": {"enabled": True, "command": command, "args": args}}, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -658,7 +658,7 @@ def test_command_and_args_overrides(self, command, args): def test_command_and_args_overrides_are_templated(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "command": ["{{ .Release.Name }}"], "args": ["{{ .Release.Service }}"], @@ -673,7 +673,7 @@ def test_command_and_args_overrides_are_templated(self): def test_should_add_component_specific_annotations(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "annotations": {"test_annotation": "test_annotation_value"}, }, @@ -686,7 +686,7 @@ def test_should_add_component_specific_annotations(self): def test_rpc_server_pod_hostaliases(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "hostAliases": [{"ip": "127.0.0.1", "hostnames": ["foo.local"]}], }, @@ -703,7 +703,7 @@ class TestRPCServerService: def test_default_service(self): docs = render_chart( - values={"rpcServer": {"enabled": True}}, + values={"_rpcServer": {"enabled": True}}, show_only=["templates/rpc-server/rpc-server-service.yaml"], ) @@ -718,8 +718,8 @@ def test_default_service(self): def test_overrides(self): docs = render_chart( values={ - "ports": {"rpcServer": 9000}, - "rpcServer": { + "ports": {"_rpcServer": 9000}, + "_rpcServer": { "enabled": True, "service": { "type": "LoadBalancer", @@ -743,13 +743,13 @@ def test_overrides(self): [ ([{"port": 8888}], [{"port": 8888}]), # name is optional with a single port ( - [{"name": "{{ .Release.Name }}", "protocol": "UDP", "port": "{{ .Values.ports.rpcServer }}"}], + [{"name": "{{ .Release.Name }}", "protocol": "UDP", "port": "{{ .Values.ports._rpcServer }}"}], [{"name": "release-name", "protocol": "UDP", "port": 9080}], ), ([{"name": "only_sidecar", "port": "{{ int 9000 }}"}], [{"name": "only_sidecar", "port": 9000}]), ( [ - {"name": "rpc-server", "port": "{{ .Values.ports.rpcServer }}"}, + {"name": "rpc-server", "port": "{{ .Values.ports._rpcServer }}"}, {"name": "sidecar", "port": 80, "targetPort": "sidecar"}, ], [ @@ -762,7 +762,7 @@ def test_overrides(self): def test_ports_overrides(self, ports, expected_ports): docs = render_chart( values={ - "rpcServer": {"enabled": True, "service": {"ports": ports}}, + "_rpcServer": {"enabled": True, "service": {"ports": ports}}, }, show_only=["templates/rpc-server/rpc-server-service.yaml"], ) @@ -772,7 +772,7 @@ def test_ports_overrides(self, ports, expected_ports): def test_should_add_component_specific_labels(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "labels": {"test_label": "test_label_value"}, }, @@ -798,7 +798,7 @@ def test_should_add_component_specific_labels(self): def test_nodeport_service(self, ports, expected_ports): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "service": { "type": "NodePort", @@ -826,7 +826,7 @@ def test_defaults(self): docs = render_chart( values={ "networkPolicies": {"enabled": True}, - "rpcServer": { + "_rpcServer": { "enabled": True, "networkPolicy": { "ingress": { @@ -851,7 +851,7 @@ def test_defaults(self): ([{"port": "sidecar"}], [{"port": "sidecar"}]), ( [ - {"port": "{{ .Values.ports.rpcServer }}"}, + {"port": "{{ .Values.ports._rpcServer }}"}, {"port": 80}, ], [ @@ -865,7 +865,7 @@ def test_ports_overrides(self, ports, expected_ports): docs = render_chart( values={ "networkPolicies": {"enabled": True}, - "rpcServer": { + "_rpcServer": { "enabled": True, "networkPolicy": { "ingress": { @@ -884,7 +884,7 @@ def test_should_add_component_specific_labels(self): docs = render_chart( values={ "networkPolicies": {"enabled": True}, - "rpcServer": { + "_rpcServer": { "enabled": True, "labels": {"test_label": "test_label_value"}, }, @@ -901,7 +901,7 @@ class TestRPCServerServiceAccount: def test_should_add_component_specific_labels(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "serviceAccount": {"create": True}, "labels": {"test_label": "test_label_value"}, @@ -915,7 +915,7 @@ def test_should_add_component_specific_labels(self): def test_default_automount_service_account_token(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "serviceAccount": {"create": True}, }, @@ -927,7 +927,7 @@ def test_default_automount_service_account_token(self): def test_overridden_automount_service_account_token(self): docs = render_chart( values={ - "rpcServer": { + "_rpcServer": { "enabled": True, "serviceAccount": {"create": True, "automountServiceAccountToken": False}, }, @@ -935,3 +935,7 @@ def test_overridden_automount_service_account_token(self): show_only=["templates/rpc-server/rpc-server-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is False + + +a = ['helm', 'template', 'release-name', '/Users/dstandish/code/airflow/chart', '--values', '/var/folders/9c/tknx7xx10qx92983y1r5djb40000gn/T/tmpch5mly3e', '--kube-version', '1.29.1', '--namespace', 'default', '--show-only', 'templates/rpc-server/rpc-server-deployment.yaml'] +print(" ".join(a)) diff --git a/helm_tests/security/test_rbac.py b/helm_tests/security/test_rbac.py index ae23a5af63f88..75924684cda98 100644 --- a/helm_tests/security/test_rbac.py +++ b/helm_tests/security/test_rbac.py @@ -141,7 +141,7 @@ def test_deployments_no_rbac_no_sa(self, version): "redis": {"serviceAccount": {"create": False}}, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, - "rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, + "_rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "statsd": {"serviceAccount": {"create": False}}, @@ -168,7 +168,7 @@ def test_deployments_no_rbac_with_sa(self, version): "cleanup": {"enabled": True}, "flower": {"enabled": True}, "pgbouncer": {"enabled": True}, - "rpcServer": {"enabled": True}, + "_rpcServer": {"enabled": True}, }, version=version, ), @@ -194,7 +194,7 @@ def test_deployments_with_rbac_no_sa(self, version): }, "scheduler": {"serviceAccount": {"create": False}}, "webserver": {"serviceAccount": {"create": False}}, - "rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, + "_rpcServer": {"enabled": True, "serviceAccount": {"create": False}}, "workers": {"serviceAccount": {"create": False}}, "triggerer": {"serviceAccount": {"create": False}}, "flower": {"enabled": True, "serviceAccount": {"create": False}}, @@ -228,7 +228,7 @@ def test_deployments_with_rbac_with_sa(self, version): "cleanup": {"enabled": True}, "flower": {"enabled": True}, "pgbouncer": {"enabled": True}, - "rpcServer": {"enabled": True}, + "_rpcServer": {"enabled": True}, }, version=version, ), @@ -254,7 +254,7 @@ def test_service_account_custom_names(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, - "rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, + "_rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, @@ -291,7 +291,7 @@ def test_service_account_custom_names_in_objects(self): }, "scheduler": {"serviceAccount": {"name": CUSTOM_SCHEDULER_NAME}}, "webserver": {"serviceAccount": {"name": CUSTOM_WEBSERVER_NAME}}, - "rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, + "_rpcServer": {"enabled": True, "serviceAccount": {"name": CUSTOM_RPC_SERVER_NAME}}, "workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}}, "triggerer": {"serviceAccount": {"name": CUSTOM_TRIGGERER_NAME}}, "flower": {"enabled": True, "serviceAccount": {"name": CUSTOM_FLOWER_NAME}}, @@ -334,7 +334,7 @@ def test_service_account_without_resource(self): "redis": {"enabled": False}, "flower": {"enabled": False}, "statsd": {"enabled": False}, - "rpcServer": {"enabled": False}, + "_rpcServer": {"enabled": False}, "webserver": {"defaultUser": {"enabled": False}}, }, ) From 17d221379d6a985a62d391063a218ba6c88adfd8 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 21:05:01 -0700 Subject: [PATCH 13/21] fixup! rename rpcServer to _rpcServer and add newsfragment / docs re experimental / dev only --- chart/newsfragments/38549.significant.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 chart/newsfragments/38549.significant.rst diff --git a/chart/newsfragments/38549.significant.rst b/chart/newsfragments/38549.significant.rst new file mode 100644 index 0000000000000..5f895f0323d96 --- /dev/null +++ b/chart/newsfragments/38549.significant.rst @@ -0,0 +1,3 @@ +Experimental support for RPC server + +For development purposes only, you can enable deployment of the RPC server (AIP-44) with the node ``_rpcServer``. Development is ongoing and it may change at any time. When official support for RPC server is added in the helm chart, the node will be renamed to ``rpcServer``. From fd73a2d968e569f55bdae21d8d4cdab6a959e566 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 21:06:41 -0700 Subject: [PATCH 14/21] Print stderr from helm test failures Previously you would not see the error message; you'd only see that the command failed. Now you'll see it. --- tests/charts/helm_template_generator.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/charts/helm_template_generator.py b/tests/charts/helm_template_generator.py index 2cdcce72ce155..93d6ee6759715 100644 --- a/tests/charts/helm_template_generator.py +++ b/tests/charts/helm_template_generator.py @@ -135,7 +135,16 @@ def render_chart( if show_only: for i in show_only: command.extend(["--show-only", i]) - templates = subprocess.check_output(command, stderr=subprocess.PIPE, cwd=chart_dir) + result = subprocess.run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=chart_dir) + if result.returncode != 0: + try: + result.check_returncode() + except Exception as e: + raise RuntimeError( + "Helm command failed. Stderr: \n%s" % result.stderr.decode("utf-8") + ) from e + + templates = result.stdout k8s_objects = yaml.full_load_all(templates) k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object] # type: ignore for k8s_object in k8s_objects: From 6a1e1450e55b5b0f8f997e0c45b99df55fe95833 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 21:09:54 -0700 Subject: [PATCH 15/21] fixup! rename rpcServer to _rpcServer and add newsfragment / docs re experimental / dev only --- helm_tests/airflow_core/test_rpc_server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index 9bfb3c8e21499..93de97475b025 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -935,7 +935,3 @@ def test_overridden_automount_service_account_token(self): show_only=["templates/rpc-server/rpc-server-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is False - - -a = ['helm', 'template', 'release-name', '/Users/dstandish/code/airflow/chart', '--values', '/var/folders/9c/tknx7xx10qx92983y1r5djb40000gn/T/tmpch5mly3e', '--kube-version', '1.29.1', '--namespace', 'default', '--show-only', 'templates/rpc-server/rpc-server-deployment.yaml'] -print(" ".join(a)) From b4d0201c0f1e453adcc4ec3adc632aa6782d83b0 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 21:10:19 -0700 Subject: [PATCH 16/21] fixup! rename rpcServer to _rpcServer and add newsfragment / docs re experimental / dev only --- helm_tests/airflow_core/test_rpc_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index 93de97475b025..cfaad782ee63f 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -743,7 +743,13 @@ def test_overrides(self): [ ([{"port": 8888}], [{"port": 8888}]), # name is optional with a single port ( - [{"name": "{{ .Release.Name }}", "protocol": "UDP", "port": "{{ .Values.ports._rpcServer }}"}], + [ + { + "name": "{{ .Release.Name }}", + "protocol": "UDP", + "port": "{{ .Values.ports._rpcServer }}", + } + ], [{"name": "release-name", "protocol": "UDP", "port": 9080}], ), ([{"name": "only_sidecar", "port": "{{ int 9000 }}"}], [{"name": "only_sidecar", "port": 9000}]), From 9bb5bc0e508926de8f315004287b0af21c3eb5eb Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 21:13:04 -0700 Subject: [PATCH 17/21] disallow legacy securityContext behavior --- chart/values.schema.json | 13 --------- helm_tests/airflow_core/test_rpc_server.py | 32 +++++++++------------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index eb1d4ad63977f..0216c0cff3af9 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -4542,19 +4542,6 @@ } } }, - "securityContext": { - "description": "Security context for the RPC server job pod (deprecated, use `securityContexts` instead). If not set, the values from `securityContext` will be used.", - "type": "object", - "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", - "default": {}, - "examples": [ - { - "runAsUser": 50000, - "runAsGroup": 0, - "fsGroup": 0 - } - ] - }, "containerLifecycleHooks": { "description": "Container Lifecycle Hooks definition for the RPC server. If not set, the values from global `containerLifecycleHooks` will be used.", "type": "object", diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index cfaad782ee63f..71872232345d1 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -577,27 +577,21 @@ def test_rpc_server_security_contexts_are_configurable(self): } == jmespath.search("spec.template.spec.securityContext", docs[0]) def test_rpc_server_security_context_legacy(self): - docs = render_chart( - values={ - "_rpcServer": { - "enabled": True, - "securityContext": { - "fsGroup": 1000, - "runAsGroup": 1001, - "runAsNonRoot": True, - "runAsUser": 2000, + with pytest.raises(RuntimeError, match="Additional property securityContext is not allowed"): + render_chart( + values={ + "_rpcServer": { + "enabled": True, + "securityContext": { + "fsGroup": 1000, + "runAsGroup": 1001, + "runAsNonRoot": True, + "runAsUser": 2000, + }, }, }, - }, - show_only=["templates/rpc-server/rpc-server-deployment.yaml"], - ) - - assert { - "runAsUser": 2000, - "runAsGroup": 1001, - "fsGroup": 1000, - "runAsNonRoot": True, - } == jmespath.search("spec.template.spec.securityContext", docs[0]) + show_only=["templates/rpc-server/rpc-server-deployment.yaml"], + ) def test_rpc_server_resources_are_not_added_by_default(self): docs = render_chart( From 3eead4ce9eb79e9003a799f6860c0d0b15c78ae0 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 22:52:34 -0700 Subject: [PATCH 18/21] fixup! Print stderr from helm test failures --- tests/charts/helm_template_generator.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/charts/helm_template_generator.py b/tests/charts/helm_template_generator.py index 93d6ee6759715..d3a8ed954b65e 100644 --- a/tests/charts/helm_template_generator.py +++ b/tests/charts/helm_template_generator.py @@ -102,6 +102,11 @@ def validate_k8s_object(instance, kubernetes_version): validate.validate(instance) +class HelmFailedError(subprocess.CalledProcessError): + def __str__(self): + return f"Helm command failed. Args: {self.args}\nStderr: \n{self.stderr.decode('utf-8')}" + + def render_chart( name="release-name", values=None, @@ -135,15 +140,9 @@ def render_chart( if show_only: for i in show_only: command.extend(["--show-only", i]) - result = subprocess.run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=chart_dir) - if result.returncode != 0: - try: - result.check_returncode() - except Exception as e: - raise RuntimeError( - "Helm command failed. Stderr: \n%s" % result.stderr.decode("utf-8") - ) from e - + result = subprocess.run(command, capture_output=True, cwd=chart_dir) + if result.returncode: + raise HelmFailedError(result.returncode, result.args, result.stdout, result.stderr) templates = result.stdout k8s_objects = yaml.full_load_all(templates) k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object] # type: ignore From ba7e1e153fbc17729797a9b95204ccdbcc615c5b Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 17 May 2024 23:01:06 -0700 Subject: [PATCH 19/21] fix test --- helm_tests/airflow_core/test_rpc_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index 71872232345d1..a69efbfa25d36 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -16,6 +16,8 @@ # under the License. from __future__ import annotations +from subprocess import CalledProcessError + import jmespath import pytest @@ -577,7 +579,7 @@ def test_rpc_server_security_contexts_are_configurable(self): } == jmespath.search("spec.template.spec.securityContext", docs[0]) def test_rpc_server_security_context_legacy(self): - with pytest.raises(RuntimeError, match="Additional property securityContext is not allowed"): + with pytest.raises(CalledProcessError, match="Additional property securityContext is not allowed"): render_chart( values={ "_rpcServer": { From da175d37323ad8cc290d7ef4d44305e0f7a1a2a3 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:45:57 -0700 Subject: [PATCH 20/21] no need for baseUrl --- .../rpc-server/rpc-server-deployment.yaml | 18 +++++++-------- chart/values.schema.json | 5 ---- chart/values.yaml | 1 - helm_tests/airflow_core/test_rpc_server.py | 23 +++++++++++++++---- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/chart/templates/rpc-server/rpc-server-deployment.yaml b/chart/templates/rpc-server/rpc-server-deployment.yaml index f0fe851f8059a..4a3766c585322 100644 --- a/chart/templates/rpc-server/rpc-server-deployment.yaml +++ b/chart/templates/rpc-server/rpc-server-deployment.yaml @@ -189,12 +189,12 @@ spec: containerPort: {{ .Values.ports._rpcServer }} livenessProbe: httpGet: - path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + path: {{ if .Values.config.core.internal_api_url }}{{- with urlParse (tpl .Values.config.core.internal_api_url .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health port: {{ .Values.ports._rpcServer }} - {{- if .Values._rpcServer.baseUrl}} + {{- if .Values.config.core.internal_api_url}} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.core.internal_api_url .)).host "" }} {{- end }} scheme: {{ .Values._rpcServer.livenessProbe.scheme | default "http" }} initialDelaySeconds: {{ .Values._rpcServer.livenessProbe.initialDelaySeconds }} @@ -203,12 +203,12 @@ spec: periodSeconds: {{ .Values._rpcServer.livenessProbe.periodSeconds }} readinessProbe: httpGet: - path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + path: {{ if .Values.config.core.internal_api_url }}{{- with urlParse (tpl .Values.config.core.internal_api_url .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health port: {{ .Values.ports._rpcServer }} - {{- if .Values._rpcServer.baseUrl }} + {{- if .Values.config.core.internal_api_url }} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.core.internal_api_url .)).host "" }} {{- end }} scheme: {{ .Values._rpcServer.readinessProbe.scheme | default "http" }} initialDelaySeconds: {{ .Values._rpcServer.readinessProbe.initialDelaySeconds }} @@ -217,12 +217,12 @@ spec: periodSeconds: {{ .Values._rpcServer.readinessProbe.periodSeconds }} startupProbe: httpGet: - path: {{ if .Values._rpcServer.baseUrl }}{{- with urlParse (tpl .Values._rpcServer.baseUrl .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health + path: {{ if .Values.config.core.internal_api_url }}{{- with urlParse (tpl .Values.config.core.internal_api_url .) }}{{ .path }}{{ end }}{{ end }}/internal_api/v1/health port: {{ .Values.ports._rpcServer }} - {{- if .Values._rpcServer.baseUrl}} + {{- if .Values.config.core.internal_api_url}} httpHeaders: - name: Host - value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values._rpcServer.baseUrl .)).host "" }} + value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.core.internal_api_url .)).host "" }} {{- end }} scheme: {{ .Values._rpcServer.startupProbe.scheme | default "http" }} timeoutSeconds: {{ .Values._rpcServer.startupProbe.timeoutSeconds }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 0216c0cff3af9..c0f3c34ee7995 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -4247,11 +4247,6 @@ "type": "boolean", "default": false }, - "baseUrl": { - "description": "RPC server base URL", - "type": "string", - "default": "" - }, "configMapAnnotations": { "description": "Extra annotations to apply to the RPC server configmap.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index b8e0e951ce4b3..d0635640fc7a1 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1159,7 +1159,6 @@ migrateDatabaseJob: # rpcServer support is experimental / dev purpose only and will later be renamed _rpcServer: enabled: false - baseUrl: "" # Labels specific to workers objects and pods labels: {} diff --git a/helm_tests/airflow_core/test_rpc_server.py b/helm_tests/airflow_core/test_rpc_server.py index a69efbfa25d36..106b2d38e8152 100644 --- a/helm_tests/airflow_core/test_rpc_server.py +++ b/helm_tests/airflow_core/test_rpc_server.py @@ -40,7 +40,10 @@ def test_is_disabled_by_default(self): def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"_rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, + values={ + "_rpcServer": {"enabled": True}, + "config": {"core": {"internal_api_url": "https://example.com:21222/mypath/path"}}, + }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -56,7 +59,12 @@ def test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(sel def test_should_add_path_to_liveness_and_readiness_and_startup_probes(self): docs = render_chart( - values={"_rpcServer": {"enabled": True, "baseUrl": "https://example.com:21222/mypath/path"}}, + values={ + "_rpcServer": { + "enabled": True, + }, + "config": {"core": {"internal_api_url": "https://example.com:21222/mypath/path"}}, + }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], ) @@ -97,8 +105,11 @@ def test_revision_history_limit(self, revision_history_limit, global_revision_hi @pytest.mark.parametrize( "values", [ - {"_rpcServer": {"enabled": True, "baseUrl": ""}}, {"_rpcServer": {"enabled": True}}, + { + "_rpcServer": {"enabled": True}, + "config": {"core": {"internal_api_url": ""}}, + }, ], ) def test_should_not_contain_host_header(self, values): @@ -122,7 +133,11 @@ def test_should_use_templated_base_url_for_probes(self): values={ "_rpcServer": { "enabled": True, - "baseUrl": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path", + }, + "config": { + "core": { + "internal_api_url": "https://{{ .Release.Name }}.com:21222/mypath/{{ .Release.Name }}/path" + } }, }, show_only=["templates/rpc-server/rpc-server-deployment.yaml"], From f82b99c685f1310d624f813e4b98a10e9b413f7c Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:55:01 -0700 Subject: [PATCH 21/21] remove newsfragment --- chart/newsfragments/38549.significant.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 chart/newsfragments/38549.significant.rst diff --git a/chart/newsfragments/38549.significant.rst b/chart/newsfragments/38549.significant.rst deleted file mode 100644 index 5f895f0323d96..0000000000000 --- a/chart/newsfragments/38549.significant.rst +++ /dev/null @@ -1,3 +0,0 @@ -Experimental support for RPC server - -For development purposes only, you can enable deployment of the RPC server (AIP-44) with the node ``_rpcServer``. Development is ongoing and it may change at any time. When official support for RPC server is added in the helm chart, the node will be renamed to ``rpcServer``.