From 68572c21126cef41ca0ee3b89941d211a658d20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Thu, 19 Dec 2019 19:29:57 +0100 Subject: [PATCH 1/3] fixup! [AIRFLOW-5903] Remove serve_logs command from CLI (#6843) --- airflow/bin/cli.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 859bf3ec432a0..9325f5038aeeb 100644 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -852,11 +852,6 @@ class CLIFactory: 'help': "Start a kerberos ticket renewer", 'args': ('principal', 'keytab', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'), - }, { - 'name': 'serve_logs', - 'func': lazy_load_command('airflow.cli.commands.serve_logs_command.serve_logs'), - 'help': "Serve logs generate by worker", - 'args': tuple(), }, { 'name': 'webserver', 'func': lazy_load_command('airflow.cli.commands.webserver_command.webserver'), From 851fe331578fa023e6519648ec437eeed2b059cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Thu, 19 Dec 2019 19:39:46 +0100 Subject: [PATCH 2/3] fixup! fixup! [AIRFLOW-5903] Remove serve_logs command from CLI (#6843) --- airflow/utils/serve_logs.py | 2 +- tests/cli/commands/test_worker_command.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py index 8d9d0653e4b92..325e9f224afa3 100644 --- a/airflow/utils/serve_logs.py +++ b/airflow/utils/serve_logs.py @@ -29,7 +29,7 @@ def serve_logs(): flask_app = flask.Flask(__name__) @flask_app.route('/log/') - def serve_logs_view(filename): # pylint: disable=unused-variable, redefined-outer-name + def serve_logs_view(filename): # pylint: disable=unused-variable log_directory = os.path.expanduser(conf.get('core', 'BASE_LOG_FOLDER')) return flask.send_from_directory( log_directory, diff --git a/tests/cli/commands/test_worker_command.py b/tests/cli/commands/test_worker_command.py index 02690edc0bacb..c360929c139b8 100644 --- a/tests/cli/commands/test_worker_command.py +++ b/tests/cli/commands/test_worker_command.py @@ -68,13 +68,13 @@ def setUpClass(cls): cls.parser = cli.CLIFactory.get_parser() def test_serve_logs_on_worker_start(self): - with patch('airflow.cli.commands.worker_command.Process') as mock_popen: + with patch('airflow.cli.commands.worker_command.Process') as mock_process: args = self.parser.parse_args(['worker', '-c', '-1']) with patch('celery.platforms.check_privileges') as mock_privil: mock_privil.return_value = 0 worker_command.worker(args) - mock_popen.assert_called() + mock_process.assert_called() def test_skip_serve_logs_on_worker_start(self): with patch('airflow.cli.commands.worker_command.Process') as mock_popen: From 73536fdaa28380e5dab326e79a6bcded3a609588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Thu, 19 Dec 2019 19:58:24 +0100 Subject: [PATCH 3/3] [AIRFLOW-6310] Group CLI commands in subcommands --- UPDATING.md | 15 +++ airflow/bin/cli.py | 111 ++++++++++-------- airflow/cli/commands/config_command.py | 13 ++ .../__init__.py} | 16 +-- .../{ => run_command}/flower_command.py | 0 .../{ => run_command}/kerberos_command.py | 0 .../{ => run_command}/scheduler_command.py | 0 .../{ => run_command}/webserver_command.py | 0 .../{ => run_command}/worker_command.py | 0 tests/cli/commands/run_command/__init__.py | 19 +++ .../test_webserver_command.py | 4 +- .../{ => run_command}/test_worker_command.py | 6 +- 12 files changed, 115 insertions(+), 69 deletions(-) rename airflow/cli/commands/{rotate_fernet_key_command.py => run_command/__init__.py} (57%) rename airflow/cli/commands/{ => run_command}/flower_command.py (100%) rename airflow/cli/commands/{ => run_command}/kerberos_command.py (100%) rename airflow/cli/commands/{ => run_command}/scheduler_command.py (100%) rename airflow/cli/commands/{ => run_command}/webserver_command.py (100%) rename airflow/cli/commands/{ => run_command}/worker_command.py (100%) create mode 100644 tests/cli/commands/run_command/__init__.py rename tests/cli/commands/{ => run_command}/test_webserver_command.py (97%) rename tests/cli/commands/{ => run_command}/test_worker_command.py (96%) diff --git a/UPDATING.md b/UPDATING.md index 16c37207f91bc..29b24c4469be7 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -41,6 +41,21 @@ assists users migrating to a new version. ## Airflow Master +### Simplification of CLI commands + +Some commands have been grouped to improve UX of CLI. New commands are available according to the following +table: + +| Old command | New command | +|---------------------------|------------------------------------| +| airflow kerberos | airflow run kerberos | +| airflow webserver | airflow run webserver | +| airflow scheduler | airflow run scheduler | +| airflow worker | airflow run worker | +| airflow flower | airflow run flower | +| airflow config | airflow config show | +| airflow rotate_fernet_key | airflow config rotate_fernet_key | + ### Remove serve_logs command from CLI The ``serve_logs`` command has been deleted. This command should be run only by internal application mechanisms diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 9325f5038aeeb..26f0b1a90b484 100644 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -847,42 +847,44 @@ class CLIFactory: }, ), }, { - 'name': 'kerberos', - 'func': lazy_load_command('airflow.cli.commands.kerberos_command.kerberos'), - 'help': "Start a kerberos ticket renewer", - 'args': ('principal', 'keytab', 'pid', - 'daemon', 'stdout', 'stderr', 'log_file'), - }, { - 'name': 'webserver', - 'func': lazy_load_command('airflow.cli.commands.webserver_command.webserver'), - 'help': "Start a Airflow webserver instance", - 'args': ('port', 'workers', 'workerclass', 'worker_timeout', 'hostname', - 'pid', 'daemon', 'stdout', 'stderr', 'access_logfile', - 'error_logfile', 'log_file', 'ssl_cert', 'ssl_key', 'debug'), - }, { - 'name': 'scheduler', - 'func': lazy_load_command('airflow.cli.commands.scheduler_command.scheduler'), - 'help': "Start a scheduler instance", - 'args': ('dag_id_opt', 'subdir', 'num_runs', - 'do_pickle', 'pid', 'daemon', 'stdout', 'stderr', - 'log_file'), - }, { - 'name': 'worker', - 'func': lazy_load_command('airflow.cli.commands.worker_command.worker'), - 'help': "Start a Celery worker node", - 'args': ('do_pickle', 'queues', 'concurrency', 'celery_hostname', - 'pid', 'daemon', 'stdout', 'stderr', 'log_file', 'autoscale', 'skip_serve_logs'), - }, { - 'name': 'flower', - 'func': lazy_load_command('airflow.cli.commands.flower_command.flower'), - 'help': "Start a Celery Flower", - 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'flower_url_prefix', - 'flower_basic_auth', 'broker_api', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'), - }, { - 'name': 'version', - 'func': lazy_load_command('airflow.cli.commands.version_command.version'), - 'help': "Show the version", - 'args': tuple(), + 'help': "Run application component", + 'name': 'run', + 'subcommands': ( + { + 'name': 'kerberos', + 'func': lazy_load_command('airflow.cli.commands.run_command.kerberos_command.kerberos'), + 'help': "Start a kerberos ticket renewer", + 'args': ('principal', 'keytab', 'pid', + 'daemon', 'stdout', 'stderr', 'log_file'), + }, { + 'name': 'webserver', + 'func': lazy_load_command('airflow.cli.commands.run_command.webserver_command.webserver'), + 'help': "Start a Airflow webserver instance", + 'args': ('port', 'workers', 'workerclass', 'worker_timeout', 'hostname', + 'pid', 'daemon', 'stdout', 'stderr', 'access_logfile', + 'error_logfile', 'log_file', 'ssl_cert', 'ssl_key', 'debug'), + }, { + 'name': 'scheduler', + 'func': lazy_load_command('airflow.cli.commands.run_command.scheduler_command.scheduler'), + 'help': "Start a scheduler instance", + 'args': ('dag_id_opt', 'subdir', 'num_runs', + 'do_pickle', 'pid', 'daemon', 'stdout', 'stderr', + 'log_file'), + }, { + 'name': 'worker', + 'func': lazy_load_command('airflow.cli.commands.run_command.worker_command.worker'), + 'help': "Start a Celery worker node", + 'args': ('do_pickle', 'queues', 'concurrency', 'celery_hostname', + 'pid', 'daemon', 'stdout', 'stderr', 'log_file', 'autoscale', 'skip_serve_logs'), + }, { + 'name': 'flower', + 'func': lazy_load_command('airflow.cli.commands.run_command.flower_command.flower'), + 'help': "Start a Celery Flower", + 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'flower_url_prefix', + 'flower_basic_auth', 'broker_api', 'pid', 'daemon', 'stdout', 'stderr', + 'log_file'), + } + ) }, { 'help': "List/Add/Delete connections", 'name': 'connections', @@ -976,21 +978,30 @@ class CLIFactory: 'func': lazy_load_command('airflow.cli.commands.sync_perm_command.sync_perm'), 'help': "Update permissions for existing roles and DAGs.", 'args': tuple(), - }, - { - 'name': 'rotate_fernet_key', - 'func': lazy_load_command('airflow.cli.commands.rotate_fernet_key_command.rotate_fernet_key'), - 'help': 'Rotate all encrypted connection credentials and variables; see ' - 'https://airflow.readthedocs.io/en/stable/howto/secure-connections.html' - '#rotating-encryption-keys.', - 'args': (), - }, - { + }, { + 'help': 'Manage configuration', 'name': 'config', - 'func': lazy_load_command('airflow.cli.commands.config_command.show_config'), - 'help': 'Show current application configuration.', - 'args': (), - }, + 'subcommands': ( + { + 'func': lazy_load_command('airflow.cli.commands.config_command.show_config'), + 'name': 'show', + 'help': 'Show current application configuration.', + 'args': (), + }, { + 'func': lazy_load_command('airflow.cli.commands.config_command.rotate_fernet_key'), + 'name': 'rotate_fernet_key', + 'help': 'Rotate all encrypted connection credentials and variables; see ' + 'https://airflow.readthedocs.io/en/stable/howto/secure-connections.html' + '#rotating-encryption-keys.', + 'args': (), + }, + ), + }, { + 'name': 'version', + 'func': lazy_load_command('airflow.cli.commands.version_command.version'), + 'help': "Show the version", + 'args': tuple(), + } ) subparsers_dict = {sp.get('name') or sp['func'].__name__: sp for sp in subparsers} # type: ignore dag_subparsers = ( diff --git a/airflow/cli/commands/config_command.py b/airflow/cli/commands/config_command.py index 999f933f09149..1702d7dfc9547 100644 --- a/airflow/cli/commands/config_command.py +++ b/airflow/cli/commands/config_command.py @@ -16,6 +16,8 @@ # under the License. """Config sub-commands""" from airflow.configuration import conf +from airflow.models import Connection, Variable +from airflow.utils import cli as cli_utils, db def show_config(args): @@ -27,3 +29,14 @@ def show_config(args): for parameter_key, value in sorted(parameters.items()): print(f"{parameter_key}={value}") print() + + +@cli_utils.action_logging +def rotate_fernet_key(args): + """Rotates all encrypted connection credentials and variables""" + with db.create_session() as session: + for conn in session.query(Connection).filter( + Connection.is_encrypted | Connection.is_extra_encrypted): + conn.rotate_fernet_key() + for var in session.query(Variable).filter(Variable.is_encrypted): + var.rotate_fernet_key() diff --git a/airflow/cli/commands/rotate_fernet_key_command.py b/airflow/cli/commands/run_command/__init__.py similarity index 57% rename from airflow/cli/commands/rotate_fernet_key_command.py rename to airflow/cli/commands/run_command/__init__.py index 303812adf942c..114d189da14ab 100644 --- a/airflow/cli/commands/rotate_fernet_key_command.py +++ b/airflow/cli/commands/run_command/__init__.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,17 +16,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Rotate Fernet key command""" -from airflow.models import Connection, Variable -from airflow.utils import cli as cli_utils, db - - -@cli_utils.action_logging -def rotate_fernet_key(args): - """Rotates all encrypted connection credentials and variables""" - with db.create_session() as session: - for conn in session.query(Connection).filter( - Connection.is_encrypted | Connection.is_extra_encrypted): - conn.rotate_fernet_key() - for var in session.query(Variable).filter(Variable.is_encrypted): - var.rotate_fernet_key() diff --git a/airflow/cli/commands/flower_command.py b/airflow/cli/commands/run_command/flower_command.py similarity index 100% rename from airflow/cli/commands/flower_command.py rename to airflow/cli/commands/run_command/flower_command.py diff --git a/airflow/cli/commands/kerberos_command.py b/airflow/cli/commands/run_command/kerberos_command.py similarity index 100% rename from airflow/cli/commands/kerberos_command.py rename to airflow/cli/commands/run_command/kerberos_command.py diff --git a/airflow/cli/commands/scheduler_command.py b/airflow/cli/commands/run_command/scheduler_command.py similarity index 100% rename from airflow/cli/commands/scheduler_command.py rename to airflow/cli/commands/run_command/scheduler_command.py diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/run_command/webserver_command.py similarity index 100% rename from airflow/cli/commands/webserver_command.py rename to airflow/cli/commands/run_command/webserver_command.py diff --git a/airflow/cli/commands/worker_command.py b/airflow/cli/commands/run_command/worker_command.py similarity index 100% rename from airflow/cli/commands/worker_command.py rename to airflow/cli/commands/run_command/worker_command.py diff --git a/tests/cli/commands/run_command/__init__.py b/tests/cli/commands/run_command/__init__.py new file mode 100644 index 0000000000000..b7f8352944d3f --- /dev/null +++ b/tests/cli/commands/run_command/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/run_command/test_webserver_command.py similarity index 97% rename from tests/cli/commands/test_webserver_command.py rename to tests/cli/commands/run_command/test_webserver_command.py index 9a68b058c21e2..9d588dcfc3cf4 100644 --- a/tests/cli/commands/test_webserver_command.py +++ b/tests/cli/commands/run_command/test_webserver_command.py @@ -26,8 +26,8 @@ from airflow import settings from airflow.bin import cli -from airflow.cli.commands import webserver_command -from airflow.cli.commands.webserver_command import get_num_ready_workers_running +from airflow.cli.commands.run_command import webserver_command +from airflow.cli.commands.run_command.webserver_command import get_num_ready_workers_running from airflow.models import DagBag from airflow.utils.cli import setup_locations from tests.test_utils.config import conf_vars diff --git a/tests/cli/commands/test_worker_command.py b/tests/cli/commands/run_command/test_worker_command.py similarity index 96% rename from tests/cli/commands/test_worker_command.py rename to tests/cli/commands/run_command/test_worker_command.py index c360929c139b8..d2dfdbbb6f9d8 100644 --- a/tests/cli/commands/test_worker_command.py +++ b/tests/cli/commands/run_command/test_worker_command.py @@ -23,7 +23,7 @@ import airflow from airflow.bin import cli -from airflow.cli.commands import worker_command +from airflow.cli.commands.run_command import worker_command from tests.compat import mock, patch from tests.test_utils.config import conf_vars @@ -77,10 +77,10 @@ def test_serve_logs_on_worker_start(self): mock_process.assert_called() def test_skip_serve_logs_on_worker_start(self): - with patch('airflow.cli.commands.worker_command.Process') as mock_popen: + with patch('airflow.cli.commands.worker_command.Process') as mock_process: args = self.parser.parse_args(['worker', '-c', '-1', '-s']) with patch('celery.platforms.check_privileges') as mock_privil: mock_privil.return_value = 0 worker_command.worker(args) - mock_popen.assert_not_called() + mock_process.assert_not_called()