From d8d851d70a4413685468cf2e73415552a1d033e5 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Fri, 25 Jun 2021 09:10:06 +0100 Subject: [PATCH] Set process title for ``serve-logs`` and ``LocalExecutor`` (#16644) Follow up of https://github.com/apache/airflow/pull/16623. This PR/commits adds title to serve-logs command and multiprocessing manager for LocalExecutor. The serve-logs process is on celery worker when using CeleryExecutor but for LocalExecutor, it is a separate process in Scheduler. **Before**: ``` root 124 0.0 0.0 6676 4636 pts/1 Ss Jun23 0:00 \_ -bash root 25299 25.3 2.6 988372 326344 pts/1 Sl+ 01:30 0:09 | \_ /usr/local/bin/python /usr/local/bin/airflow webserver root 25510 3.6 0.4 121068 57152 pts/1 S+ 01:31 0:00 | \_ gunicorn: master [airflow-webserver] root 25555 35.7 2.5 983584 316564 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25556 35.7 2.5 983840 316684 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25557 35.5 2.5 983840 316548 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25558 37.2 2.5 984920 317700 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 128 0.0 0.0 6676 4552 pts/2 Ss Jun23 0:00 \_ -bash root 25090 5.8 0.9 467508 118808 pts/2 S+ 01:30 0:03 \_ /usr/local/bin/python /usr/local/bin/airflow scheduler root 25098 0.0 0.7 466080 97800 pts/2 S+ 01:30 0:00 \_ /usr/local/bin/python /usr/local/bin/airflow scheduler root 25099 0.4 0.8 1391812 99788 pts/2 Sl+ 01:30 0:00 \_ /usr/local/bin/python /usr/local/bin/airflow scheduler root 25107 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25109 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25114 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25117 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25120 0.0 0.8 466080 98552 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25125 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--lo root 26139 0.0 0.8 468988 102204 pts/2 S+ 01:31 0:00 | \_ airflow task supervisor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--local' root 25128 0.0 0.7 466080 98076 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25132 0.0 0.7 466080 98076 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25137 1.1 0.8 466592 100016 pts/2 S 01:30 0:00 \_ airflow scheduler -- DagFileProcessorManager root@a7c8aa590704:/opt/airflow# ps aux ``` **After**: ``` root 124 0.0 0.0 6676 4636 pts/1 Ss Jun23 0:00 \_ -bash root 25299 25.3 2.6 988372 326344 pts/1 Sl+ 01:30 0:09 | \_ /usr/local/bin/python /usr/local/bin/airflow webserver root 25510 3.6 0.4 121068 57152 pts/1 S+ 01:31 0:00 | \_ gunicorn: master [airflow-webserver] root 25555 35.7 2.5 983584 316564 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25556 35.7 2.5 983840 316684 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25557 35.5 2.5 983840 316548 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 25558 37.2 2.5 984920 317700 pts/1 Sl+ 01:31 0:08 | \_ [ready] gunicorn: worker [airflow-webserver] root 128 0.0 0.0 6676 4552 pts/2 Ss Jun23 0:00 \_ -bash root 25090 5.8 0.9 467508 118808 pts/2 S+ 01:30 0:03 \_ /usr/local/bin/python /usr/local/bin/airflow scheduler root 25098 0.0 0.7 466080 97800 pts/2 S+ 01:30 0:00 \_ airflow serve-logs root 25099 0.4 0.8 1391812 99788 pts/2 Sl+ 01:30 0:00 \_ airflow executor -- LocalExecutor root 25107 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25109 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25114 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25117 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25120 0.0 0.8 466080 98552 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25125 0.0 0.8 466080 98548 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--lo root 26139 0.0 0.8 468988 102204 pts/2 S+ 01:31 0:00 | \_ airflow task supervisor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--local' root 25128 0.0 0.7 466080 98076 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25132 0.0 0.7 466080 98076 pts/2 S+ 01:30 0:00 \_ airflow worker -- LocalExecutor root 25137 1.1 0.8 466592 100016 pts/2 S 01:30 0:00 \_ airflow scheduler -- DagFileProcessorManager root@a7c8aa590704:/opt/airflow# ps aux ``` (cherry picked from commit 1b2535e442599d4607eb6497864af26ad5afa807) --- airflow/executors/local_executor.py | 5 ++++- airflow/utils/serve_logs.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py index a29342f2c8e46..b4848ed15ebe0 100644 --- a/airflow/executors/local_executor.py +++ b/airflow/executors/local_executor.py @@ -31,7 +31,7 @@ from queue import Empty, Queue # pylint: disable=unused-import # noqa: F401 from typing import Any, List, Optional, Tuple, Union # pylint: disable=unused-import # noqa: F401 -from setproctitle import setproctitle # pylint: disable=no-name-in-module +from setproctitle import getproctitle, setproctitle # pylint: disable=no-name-in-module from airflow import settings from airflow.exceptions import AirflowException @@ -340,7 +340,10 @@ def end(self): def start(self) -> None: """Starts the executor""" + old_proctitle = getproctitle() + setproctitle("airflow executor -- LocalExecutor") self.manager = Manager() + setproctitle(old_proctitle) self.result_queue = self.manager.Queue() self.workers = [] self.workers_used = 0 diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py index 0fefa420b7d84..e7890f7810186 100644 --- a/airflow/utils/serve_logs.py +++ b/airflow/utils/serve_logs.py @@ -19,6 +19,7 @@ import os import flask +from setproctitle import setproctitle from airflow.configuration import conf @@ -27,6 +28,7 @@ def serve_logs(): """Serves logs generated by Worker""" print("Starting flask") flask_app = flask.Flask(__name__) + setproctitle("airflow serve-logs") @flask_app.route('/log/') def serve_logs_view(filename): # pylint: disable=unused-variable