From 2d238843c628234f7883f73b67a9f5ee092dd232 Mon Sep 17 00:00:00 2001 From: Pavel Salnikov <90701144+pavel-salnikov@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:39:08 +0200 Subject: [PATCH 1/2] Remove templated config_file path, fix parameter comments --- .../workflows/kubernetes_pod_operator_c2.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/composer/workflows/kubernetes_pod_operator_c2.py b/composer/workflows/kubernetes_pod_operator_c2.py index 3b0f5ee2155..a7b622f08ea 100644 --- a/composer/workflows/kubernetes_pod_operator_c2.py +++ b/composer/workflows/kubernetes_pod_operator_c2.py @@ -63,7 +63,7 @@ schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY, ) as dag: - # Only name, namespace, image, and task_id are required to create a + # Only name, image, and task_id are required to create a # KubernetesPodOperator. In Cloud Composer, the config file found at # `/home/airflow/composer_kube_config` contains credentials for # Cloud Composer's Google Kubernetes Engine cluster that is created @@ -75,11 +75,12 @@ # Name of task you want to run, used to generate Pod ID. name="pod-ex-minimum", # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. The cmds parameter is templated. + # entrypoint is used. cmds=["echo"], # The namespace to run within Kubernetes. In Composer 2 environments # after December 2022, the default namespace is - # `composer-user-workloads`. + # `composer-user-workloads`. Always use the + # `composer-user-workloads` namespace with Composer 3. namespace="composer-user-workloads", # Docker image specified. Defaults to hub.docker.com, but any fully # qualified URLs will point to a custom repository. Supports private @@ -88,7 +89,7 @@ # uses has permission to access the Google Container Registry # (the default service account has permission) image="gcr.io/gcp-runtimes/ubuntu_20_0_4", - # Specifies path to kubernetes config. The config_file is templated. + # Specifies path to Kubernetes config. config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", @@ -100,11 +101,12 @@ name="ex-kube-templates", namespace="composer-user-workloads", image="bash", - # All parameters below are able to be templated with jinja -- cmds, - # arguments, env_vars, and config_file. For more information visit: - # https://airflow.apache.org/docs/apache-airflow/stable/macros-ref.html + # All parameters below can be templated with Jinja. For more information + # and the list of variables available in Airflow, see + # the Airflow templates reference: + # https://airflow.apache.org/docs/apache-airflow/stable/templates-ref.html # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. The cmds parameter is templated. + # entrypoint is used. cmds=["echo"], # DS in jinja is the execution date as YYYY-MM-DD, this docker image # will echo the execution date. Arguments to the entrypoint. The docker @@ -116,11 +118,8 @@ # setting the environment variable `MY_VALUE`. The pod will fail if # `my_value` is not set in the Airflow UI. env_vars={"MY_VALUE": "{{ var.value.my_value }}"}, - # Sets the config file to a kubernetes config file specified in - # airflow.cfg. If the configuration file does not exist or does - # not provide validcredentials the pod will fail to launch. If not - # specified, config_file defaults to ~/.kube/config - config_file="{{ conf.get('core', 'kube_config') }}", + # Specifies path to Kubernetes config. + config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", ) @@ -137,12 +136,12 @@ secrets=[secret_env, secret_volume], cmds=["echo"], # env_vars allows you to specify environment variables for your - # container to use. env_vars is templated. + # container to use. env_vars={ "EXAMPLE_VAR": "/example/value", "GOOGLE_APPLICATION_CREDENTIALS": "/var/secrets/google/service-account.json", }, - # Specifies path to kubernetes config. The config_file is templated. + # Specifies path to Kubernetes config. config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", @@ -155,10 +154,10 @@ namespace="composer-user-workloads", image="perl:5.34.0", # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. The cmds parameter is templated. + # entrypoint is used. cmds=["perl"], # Arguments to the entrypoint. The docker image's CMD is used if this - # is not provided. The arguments parameter is templated. + # is not provided. arguments=["-Mbignum=bpi", "-wle", "print bpi(2000)"], # The secrets to pass to Pod, the Pod will fail to create if the # secrets you specify in a Secret object do not exist in Kubernetes. @@ -168,7 +167,6 @@ # Timeout to start up the Pod, default is 600. startup_timeout_seconds=600, # The environment variables to be initialized in the container - # env_vars are templated. env_vars={"EXAMPLE_VAR": "/example/value"}, # If true, logs stdout output of container. Defaults to True. get_logs=True, @@ -192,7 +190,7 @@ requests={"cpu": "1000m", "memory": "10G", "ephemeral-storage": "10G"}, limits={"cpu": "1000m", "memory": "10G", "ephemeral-storage": "10G"}, ), - # Specifies path to kubernetes config. The config_file is templated. + # Specifies path to Kubernetes config. config_file="/home/airflow/composer_kube_config", # If true, the content of /airflow/xcom/return.json from container will # also be pushed to an XCom when the container ends. From ad789b9fc075fe7e1fdac5f07af905370644eea6 Mon Sep 17 00:00:00 2001 From: Pavel Salnikov <90701144+pavel-salnikov@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:21:19 +0200 Subject: [PATCH 2/2] Add back templated params info, capitalize Docker --- .../workflows/kubernetes_pod_operator_c2.py | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/composer/workflows/kubernetes_pod_operator_c2.py b/composer/workflows/kubernetes_pod_operator_c2.py index a7b622f08ea..94e3655cc44 100644 --- a/composer/workflows/kubernetes_pod_operator_c2.py +++ b/composer/workflows/kubernetes_pod_operator_c2.py @@ -75,7 +75,7 @@ # Name of task you want to run, used to generate Pod ID. name="pod-ex-minimum", # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. + # entrypoint is used. The cmds parameter is templated. cmds=["echo"], # The namespace to run within Kubernetes. In Composer 2 environments # after December 2022, the default namespace is @@ -89,7 +89,7 @@ # uses has permission to access the Google Container Registry # (the default service account has permission) image="gcr.io/gcp-runtimes/ubuntu_20_0_4", - # Specifies path to Kubernetes config. + # Specifies path to kubernetes config. The config_file is templated. config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", @@ -106,19 +106,20 @@ # the Airflow templates reference: # https://airflow.apache.org/docs/apache-airflow/stable/templates-ref.html # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. + # entrypoint is used. The cmds parameter is templated. cmds=["echo"], - # DS in jinja is the execution date as YYYY-MM-DD, this docker image - # will echo the execution date. Arguments to the entrypoint. The docker + # DS in Jinja is the execution date as YYYY-MM-DD, this Docker image + # will echo the execution date. Arguments to the entrypoint. The Docker # image's CMD is used if this is not provided. The arguments parameter # is templated. arguments=["{{ ds }}"], # The var template variable allows you to access variables defined in # Airflow UI. In this case we are getting the value of my_value and # setting the environment variable `MY_VALUE`. The pod will fail if - # `my_value` is not set in the Airflow UI. + # `my_value` is not set in the Airflow UI. The env_vars parameter + # is templated. env_vars={"MY_VALUE": "{{ var.value.my_value }}"}, - # Specifies path to Kubernetes config. + # Specifies path to Kubernetes config. The config_file is templated. config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", @@ -134,14 +135,16 @@ # The secrets to pass to Pod, the Pod will fail to create if the # secrets you specify in a Secret object do not exist in Kubernetes. secrets=[secret_env, secret_volume], + # Entrypoint of the container, if not specified the Docker container's + # entrypoint is used. The cmds parameter is templated. cmds=["echo"], # env_vars allows you to specify environment variables for your - # container to use. + # container to use. The env_vars parameter is templated. env_vars={ "EXAMPLE_VAR": "/example/value", "GOOGLE_APPLICATION_CREDENTIALS": "/var/secrets/google/service-account.json", }, - # Specifies path to Kubernetes config. + # Specifies path to kubernetes config. The config_file is templated. config_file="/home/airflow/composer_kube_config", # Identifier of connection that should be used kubernetes_conn_id="kubernetes_default", @@ -154,10 +157,10 @@ namespace="composer-user-workloads", image="perl:5.34.0", # Entrypoint of the container, if not specified the Docker container's - # entrypoint is used. + # entrypoint is used. The cmds parameter is templated. cmds=["perl"], - # Arguments to the entrypoint. The docker image's CMD is used if this - # is not provided. + # Arguments to the entrypoint. The Docker image's CMD is used if this + # is not provided. The arguments parameter is templated. arguments=["-Mbignum=bpi", "-wle", "print bpi(2000)"], # The secrets to pass to Pod, the Pod will fail to create if the # secrets you specify in a Secret object do not exist in Kubernetes. @@ -166,7 +169,8 @@ labels={"pod-label": "label-name"}, # Timeout to start up the Pod, default is 600. startup_timeout_seconds=600, - # The environment variables to be initialized in the container + # The environment variables to be initialized in the container. + # The env_vars parameter is templated. env_vars={"EXAMPLE_VAR": "/example/value"}, # If true, logs stdout output of container. Defaults to True. get_logs=True, @@ -190,7 +194,7 @@ requests={"cpu": "1000m", "memory": "10G", "ephemeral-storage": "10G"}, limits={"cpu": "1000m", "memory": "10G", "ephemeral-storage": "10G"}, ), - # Specifies path to Kubernetes config. + # Specifies path to kubernetes config. The config_file is templated. config_file="/home/airflow/composer_kube_config", # If true, the content of /airflow/xcom/return.json from container will # also be pushed to an XCom when the container ends.