Skip to content

Commit

Permalink
Set process title for serve-logs and LocalExecutor (#16644)
Browse files Browse the repository at this point in the history
Follow up of apache/airflow#16623.
This PR/commits adds title to serve-logs command and multiprocessing
manager for LocalExecutor.

The serve-logs process is on celery worker when using CeleryExecutor but
for LocalExecutor, it is a separate process in Scheduler.

**Before**:
```
root       124  0.0  0.0   6676  4636 pts/1    Ss   Jun23   0:00  \_ -bash
root     25299 25.3  2.6 988372 326344 pts/1   Sl+  01:30   0:09  |   \_ /usr/local/bin/python /usr/local/bin/airflow webserver
root     25510  3.6  0.4 121068 57152 pts/1    S+   01:31   0:00  |       \_ gunicorn: master [airflow-webserver]
root     25555 35.7  2.5 983584 316564 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25556 35.7  2.5 983840 316684 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25557 35.5  2.5 983840 316548 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25558 37.2  2.5 984920 317700 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root       128  0.0  0.0   6676  4552 pts/2    Ss   Jun23   0:00  \_ -bash
root     25090  5.8  0.9 467508 118808 pts/2   S+   01:30   0:03      \_ /usr/local/bin/python /usr/local/bin/airflow scheduler
root     25098  0.0  0.7 466080 97800 pts/2    S+   01:30   0:00          \_ /usr/local/bin/python /usr/local/bin/airflow scheduler
root     25099  0.4  0.8 1391812 99788 pts/2   Sl+  01:30   0:00          \_ /usr/local/bin/python /usr/local/bin/airflow scheduler
root     25107  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25109  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25114  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25117  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25120  0.0  0.8 466080 98552 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25125  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--lo
root     26139  0.0  0.8 468988 102204 pts/2   S+   01:31   0:00          |   \_ airflow task supervisor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--local'
root     25128  0.0  0.7 466080 98076 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25132  0.0  0.7 466080 98076 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25137  1.1  0.8 466592 100016 pts/2   S    01:30   0:00          \_ airflow scheduler -- DagFileProcessorManager
root@a7c8aa590704:/opt/airflow# ps aux
```

**After**:
```
root       124  0.0  0.0   6676  4636 pts/1    Ss   Jun23   0:00  \_ -bash
root     25299 25.3  2.6 988372 326344 pts/1   Sl+  01:30   0:09  |   \_ /usr/local/bin/python /usr/local/bin/airflow webserver
root     25510  3.6  0.4 121068 57152 pts/1    S+   01:31   0:00  |       \_ gunicorn: master [airflow-webserver]
root     25555 35.7  2.5 983584 316564 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25556 35.7  2.5 983840 316684 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25557 35.5  2.5 983840 316548 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root     25558 37.2  2.5 984920 317700 pts/1   Sl+  01:31   0:08  |           \_ [ready] gunicorn: worker [airflow-webserver]
root       128  0.0  0.0   6676  4552 pts/2    Ss   Jun23   0:00  \_ -bash
root     25090  5.8  0.9 467508 118808 pts/2   S+   01:30   0:03      \_ /usr/local/bin/python /usr/local/bin/airflow scheduler
root     25098  0.0  0.7 466080 97800 pts/2    S+   01:30   0:00          \_ airflow serve-logs
root     25099  0.4  0.8 1391812 99788 pts/2   Sl+  01:30   0:00          \_ airflow executor -- LocalExecutor
root     25107  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25109  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25114  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25117  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25120  0.0  0.8 466080 98552 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25125  0.0  0.8 466080 98548 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--lo
root     26139  0.0  0.8 468988 102204 pts/2   S+   01:31   0:00          |   \_ airflow task supervisor: ['airflow', 'tasks', 'run', 'example_bash_operator', 'run_after_loop', '2021-06-24T01:31:30.507415+00:00', '--local'
root     25128  0.0  0.7 466080 98076 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25132  0.0  0.7 466080 98076 pts/2    S+   01:30   0:00          \_ airflow worker -- LocalExecutor
root     25137  1.1  0.8 466592 100016 pts/2   S    01:30   0:00          \_ airflow scheduler -- DagFileProcessorManager
root@a7c8aa590704:/opt/airflow# ps aux
```

GitOrigin-RevId: 1b2535e442599d4607eb6497864af26ad5afa807
  • Loading branch information
kaxil authored and Cloud Composer Team committed Aug 27, 2022
1 parent 6b2eff0 commit c7df92c
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 1 deletion.
5 changes: 4 additions & 1 deletion airflow/executors/local_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from queue import Empty, Queue # pylint: disable=unused-import # noqa: F401
from typing import Any, List, Optional, Tuple, Union # pylint: disable=unused-import # noqa: F401

from setproctitle import setproctitle # pylint: disable=no-name-in-module
from setproctitle import getproctitle, setproctitle # pylint: disable=no-name-in-module

from airflow import settings
from airflow.exceptions import AirflowException
Expand Down Expand Up @@ -344,7 +344,10 @@ def end(self):

def start(self) -> None:
"""Starts the executor"""
old_proctitle = getproctitle()
setproctitle("airflow executor -- LocalExecutor")
self.manager = Manager()
setproctitle(old_proctitle)
self.result_queue = self.manager.Queue()
self.workers = []
self.workers_used = 0
Expand Down
2 changes: 2 additions & 0 deletions airflow/utils/serve_logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import os

import flask
from setproctitle import setproctitle

from airflow.configuration import conf

Expand All @@ -27,6 +28,7 @@ def serve_logs():
"""Serves logs generated by Worker"""
print("Starting flask")
flask_app = flask.Flask(__name__)
setproctitle("airflow serve-logs")

@flask_app.route('/log/<path:filename>')
def serve_logs_view(filename): # pylint: disable=unused-variable
Expand Down

0 comments on commit c7df92c

Please sign in to comment.