From 7d21301cea6be706d8336d5b08f173451de22849 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Wed, 17 Apr 2024 10:30:54 -0700 Subject: [PATCH 1/5] Allow importing the aws executors with a shorter path Import the aws executors in the __init__.py module of their respective directories so that they can be imported with a shorter and more natural looking module path. For example, this modules path: `airflow.providers.amazon.aws.executors.batch.batch_executor.AwsBatchExecutor` Becomes: `airflow.providers.amazon.aws.executors.batch.AwsBatchExecutor` --- airflow/providers/amazon/aws/executors/batch/__init__.py | 5 +++++ airflow/providers/amazon/aws/executors/ecs/__init__.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/airflow/providers/amazon/aws/executors/batch/__init__.py b/airflow/providers/amazon/aws/executors/batch/__init__.py index 13a83393a9124..5eb8e170c0981 100644 --- a/airflow/providers/amazon/aws/executors/batch/__init__.py +++ b/airflow/providers/amazon/aws/executors/batch/__init__.py @@ -14,3 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor + +AwsBatchExecutor = AwsBatchExecutor diff --git a/airflow/providers/amazon/aws/executors/ecs/__init__.py b/airflow/providers/amazon/aws/executors/ecs/__init__.py index 13a83393a9124..d41aa7e095bc0 100644 --- a/airflow/providers/amazon/aws/executors/ecs/__init__.py +++ b/airflow/providers/amazon/aws/executors/ecs/__init__.py @@ -14,3 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor + +AwsEcsExecutor = AwsEcsExecutor From 9da925e82b374b50bf5b8f0b90a089d1514cb06a Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Wed, 17 Apr 2024 11:34:00 -0700 Subject: [PATCH 2/5] Add comments to explain precommit shenanigans --- airflow/providers/amazon/aws/executors/batch/__init__.py | 5 ++++- airflow/providers/amazon/aws/executors/ecs/__init__.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/executors/batch/__init__.py b/airflow/providers/amazon/aws/executors/batch/__init__.py index 5eb8e170c0981..df80f2bde31d1 100644 --- a/airflow/providers/amazon/aws/executors/batch/__init__.py +++ b/airflow/providers/amazon/aws/executors/batch/__init__.py @@ -14,8 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations +from __future__ import annotations # Added by precommit hooks from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor +# precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" +# One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa +# type ignore of the issue. AwsBatchExecutor = AwsBatchExecutor diff --git a/airflow/providers/amazon/aws/executors/ecs/__init__.py b/airflow/providers/amazon/aws/executors/ecs/__init__.py index d41aa7e095bc0..08591993c04eb 100644 --- a/airflow/providers/amazon/aws/executors/ecs/__init__.py +++ b/airflow/providers/amazon/aws/executors/ecs/__init__.py @@ -14,8 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations +from __future__ import annotations # Added by precommit hooks from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor +# precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" +# One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa +# type ignore of the issue. AwsEcsExecutor = AwsEcsExecutor From 10872c56cd29d0cdbdd42e6511d433fe3c066a4d Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Wed, 17 Apr 2024 15:55:48 -0700 Subject: [PATCH 3/5] Reformat to make sphinx happy --- airflow/providers/amazon/aws/executors/batch/__init__.py | 5 +++-- airflow/providers/amazon/aws/executors/ecs/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airflow/providers/amazon/aws/executors/batch/__init__.py b/airflow/providers/amazon/aws/executors/batch/__init__.py index df80f2bde31d1..83c6f075031bc 100644 --- a/airflow/providers/amazon/aws/executors/batch/__init__.py +++ b/airflow/providers/amazon/aws/executors/batch/__init__.py @@ -16,9 +16,10 @@ # under the License. from __future__ import annotations # Added by precommit hooks -from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor +# from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor +from airflow.providers.amazon.aws.executors.batch import batch_executor # precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" # One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa # type ignore of the issue. -AwsBatchExecutor = AwsBatchExecutor +AwsBatchExecutor = batch_executor.AwsBatchExecutor diff --git a/airflow/providers/amazon/aws/executors/ecs/__init__.py b/airflow/providers/amazon/aws/executors/ecs/__init__.py index 08591993c04eb..41c594ad94050 100644 --- a/airflow/providers/amazon/aws/executors/ecs/__init__.py +++ b/airflow/providers/amazon/aws/executors/ecs/__init__.py @@ -16,9 +16,9 @@ # under the License. from __future__ import annotations # Added by precommit hooks -from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor +from airflow.providers.amazon.aws.executors.ecs import ecs_executor # precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" # One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa # type ignore of the issue. -AwsEcsExecutor = AwsEcsExecutor +AwsEcsExecutor = ecs_executor.AwsEcsExecutor From 6f43d9d79eeb6066b13ea1a6d06d8b81cdf9a12a Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 18 Apr 2024 10:44:58 -0700 Subject: [PATCH 4/5] Add tests for short import path --- .../amazon/aws/executors/batch/test_batch_executor.py | 5 +++++ .../providers/amazon/aws/executors/ecs/test_ecs_executor.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/providers/amazon/aws/executors/batch/test_batch_executor.py b/tests/providers/amazon/aws/executors/batch/test_batch_executor.py index e8ad0e4592cc1..8a5773bdca90a 100644 --- a/tests/providers/amazon/aws/executors/batch/test_batch_executor.py +++ b/tests/providers/amazon/aws/executors/batch/test_batch_executor.py @@ -834,3 +834,8 @@ def test_submit_job_kwargs_exec_config_overrides( final_run_task_kwargs = executor._submit_job_kwargs(mock_ti_key, command, "queue", exec_config) assert final_run_task_kwargs == expected_result + + def test_short_import_path(self): + from airflow.providers.amazon.aws.executors.batch import AwsBatchExecutor as AwsBatchExecutorShortPath + + assert AwsBatchExecutor is AwsBatchExecutorShortPath diff --git a/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py b/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py index 4110483162139..fd7bf6772620a 100644 --- a/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py +++ b/tests/providers/amazon/aws/executors/ecs/test_ecs_executor.py @@ -1703,3 +1703,8 @@ def test_run_task_kwargs_exec_config_overrides( final_run_task_kwargs = executor._run_task_kwargs(mock_ti_key, command, "queue", exec_config) assert final_run_task_kwargs == expected_result + + def test_short_import_path(self): + from airflow.providers.amazon.aws.executors.ecs import AwsEcsExecutor as AwsEcsExecutorShortPath + + assert AwsEcsExecutor is AwsEcsExecutorShortPath From 825b7d7ed6b1b4d5ba9e42344346c431b93a6cf6 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 18 Apr 2024 15:02:00 -0700 Subject: [PATCH 5/5] Implement using __all__ --- airflow/providers/amazon/aws/executors/batch/__init__.py | 8 ++------ airflow/providers/amazon/aws/executors/ecs/__init__.py | 7 ++----- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/airflow/providers/amazon/aws/executors/batch/__init__.py b/airflow/providers/amazon/aws/executors/batch/__init__.py index 83c6f075031bc..07dd9b5fd757a 100644 --- a/airflow/providers/amazon/aws/executors/batch/__init__.py +++ b/airflow/providers/amazon/aws/executors/batch/__init__.py @@ -16,10 +16,6 @@ # under the License. from __future__ import annotations # Added by precommit hooks -# from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor -from airflow.providers.amazon.aws.executors.batch import batch_executor +__all__ = ["AwsBatchExecutor"] -# precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" -# One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa -# type ignore of the issue. -AwsBatchExecutor = batch_executor.AwsBatchExecutor +from airflow.providers.amazon.aws.executors.batch.batch_executor import AwsBatchExecutor diff --git a/airflow/providers/amazon/aws/executors/ecs/__init__.py b/airflow/providers/amazon/aws/executors/ecs/__init__.py index 41c594ad94050..a8ec64d84b3d9 100644 --- a/airflow/providers/amazon/aws/executors/ecs/__init__.py +++ b/airflow/providers/amazon/aws/executors/ecs/__init__.py @@ -16,9 +16,6 @@ # under the License. from __future__ import annotations # Added by precommit hooks -from airflow.providers.amazon.aws.executors.ecs import ecs_executor +__all__ = ["AwsEcsExecutor"] -# precommit hooks (rust as of the time of commit) throws F401 - "Module imported but unused" -# One of the solutions it suggests is using a "redundant alias". This is used below instead of doing a #no-qa -# type ignore of the issue. -AwsEcsExecutor = ecs_executor.AwsEcsExecutor +from airflow.providers.amazon.aws.executors.ecs.ecs_executor import AwsEcsExecutor