Skip to content

Commit

Permalink
Resolve deprecations in Python sensors/operators/decorators (#39241)
Browse files Browse the repository at this point in the history
  • Loading branch information
Taragolis authored May 3, 2024
1 parent 8f8f456 commit 0a98435
Show file tree
Hide file tree
Showing 4 changed files with 6 additions and 58 deletions.
2 changes: 2 additions & 0 deletions tests/decorators/test_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,7 @@ def return_dict(number: int):
start_date=timezone.utcnow(),
execution_date=DEFAULT_DATE,
state=State.RUNNING,
data_interval=self.dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE),
)

ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
Expand Down Expand Up @@ -506,6 +507,7 @@ def add_num(number: int, num2: int = 2):
start_date=timezone.utcnow(),
execution_date=DEFAULT_DATE,
state=State.RUNNING,
data_interval=self.dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE),
)

bigger_number.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
Expand Down
6 changes: 3 additions & 3 deletions tests/decorators/test_python_virtualenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
pytestmark = pytest.mark.db_test

DEFAULT_DATE = timezone.datetime(2016, 1, 1)
PYTHON_VERSION = sys.version_info[0]
PYTHON_VERSION = f"{sys.version_info.major}{sys.version_info.minor}"


class TestPythonVirtualenvDecorator:
Expand Down Expand Up @@ -252,7 +252,7 @@ def f():
ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)

def test_python_3_cloudpickle(self, dag_maker):
@task.virtualenv(python_version=3, use_cloudpickle=False, requirements=["cloudpickle"])
@task.virtualenv(python_version="3", use_cloudpickle=False, requirements=["cloudpickle"])
def f():
import sys

Expand All @@ -269,7 +269,7 @@ def f():
ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)

def test_python_3_dill(self, dag_maker):
@task.virtualenv(python_version=3, use_dill=False, requirements=["dill"])
@task.virtualenv(python_version="3", use_dill=False, requirements=["dill"])
def f():
import sys

Expand Down
55 changes: 0 additions & 55 deletions tests/deprecations_ignore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,17 +184,6 @@
- tests/operators/test_datetime.py::TestBranchDateTimeOperator::test_no_target_time
- tests/operators/test_latest_only_operator.py::TestLatestOnlyOperator::test_not_skipping_external
- tests/operators/test_latest_only_operator.py::TestLatestOnlyOperator::test_skipping_non_latest
- tests/operators/test_python.py::TestBranchOperator::test_clear_skipped_downstream_task
- tests/operators/test_python.py::TestBranchOperator::test_empty_branch
- tests/operators/test_python.py::TestBranchOperator::test_with_dag_run
- tests/operators/test_python.py::TestBranchOperator::test_with_skip_in_branch_downstream_dependencies
- tests/operators/test_python.py::TestBranchOperator::test_with_skip_in_branch_downstream_dependencies2
- tests/operators/test_python.py::TestBranchOperator::test_xcom_push
- tests/operators/test_python.py::TestShortCircuitOperator::test_clear_skipped_downstream_task
- tests/operators/test_python.py::TestShortCircuitOperator::test_mapped_xcom_push_skipped_tasks
- tests/operators/test_python.py::TestShortCircuitOperator::test_short_circuiting
- tests/operators/test_python.py::TestShortCircuitOperator::test_xcom_push
- tests/operators/test_python.py::TestShortCircuitOperator::test_xcom_push_skipped_tasks
- tests/operators/test_subdag_operator.py::TestSubDagOperator::test_execute_create_dagrun_wait_until_success
- tests/operators/test_subdag_operator.py::TestSubDagOperator::test_execute_create_dagrun_with_conf
- tests/operators/test_subdag_operator.py::TestSubDagOperator::test_execute_dagrun_failed
Expand Down Expand Up @@ -227,29 +216,6 @@
- tests/operators/test_weekday.py::TestBranchDayOfWeekOperator::test_branch_xcom_push_true_branch


# BranchPythonVenv
- tests/operators/test_python.py::TestBranchPythonVirtualenvOperator::test_clear_skipped_downstream_task
- tests/operators/test_python.py::TestBranchPythonVirtualenvOperator::test_with_dag_run
- tests/operators/test_python.py::TestBranchPythonVirtualenvOperator::test_with_skip_in_branch_downstream_dependencies
- tests/operators/test_python.py::TestBranchPythonVirtualenvOperator::test_with_skip_in_branch_downstream_dependencies2
- tests/operators/test_python.py::TestBranchPythonVirtualenvOperator::test_xcom_push


# BranchExternalPython
- tests/operators/test_python.py::TestBranchExternalPythonOperator::test_clear_skipped_downstream_task
- tests/operators/test_python.py::TestBranchExternalPythonOperator::test_with_dag_run
- tests/operators/test_python.py::TestBranchExternalPythonOperator::test_with_skip_in_branch_downstream_dependencies
- tests/operators/test_python.py::TestBranchExternalPythonOperator::test_with_skip_in_branch_downstream_dependencies2
- tests/operators/test_python.py::TestBranchExternalPythonOperator::test_xcom_push


# PythonVenv


# PlainAsserts
- tests/operators/test_python.py::TestPythonVirtualenvOperator::test_airflow_context


# Serialization
- tests/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_custom_dep_detector
- tests/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_dag_params_roundtrip
Expand Down Expand Up @@ -333,27 +299,6 @@
- tests/dag_processing/test_processor.py::TestDagFileProcessor::test_execute_on_failure_callbacks_without_dag
- tests/dag_processing/test_processor.py::TestDagFileProcessor::test_failure_callbacks_should_not_drop_hostname
- tests/dag_processing/test_processor.py::TestDagFileProcessor::test_process_file_should_failure_callback
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_fail_multiple_outputs_key_type
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_fail_multiple_outputs_no_dict
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_manual_multiple_outputs_false_with_typings
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_multiple_outputs
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_multiple_outputs_empty_dict
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_multiple_outputs_ignore_typing
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_multiple_outputs_return_none
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_python_callable_arguments_are_templatized
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_python_callable_keyword_arguments_are_templatized
- tests/decorators/test_python.py::TestAirflowTaskDecorator::test_xcom_arg
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_no_system_site_packages
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_python_3_cloudpickle
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_python_3_dill
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_system_site_packages_cloudpickle
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_system_site_packages_dill
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_unpinned_requirements_cloudpickle
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_unpinned_requirements_dill
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_with_requirements_file_cloudpickle
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_with_requirements_file_dill
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_with_requirements_pinned_cloudpickle
- tests/decorators/test_python_virtualenv.py::TestPythonVirtualenvDecorator::test_with_requirements_pinned_dill
- tests/sensors/test_external_task_sensor.py::TestExternalTaskSensor::test_external_dag_sensor
- tests/sensors/test_external_task_sensor.py::TestExternalTaskSensor::test_external_dag_sensor_log
- tests/sensors/test_external_task_sensor.py::TestExternalTaskSensor::test_external_dag_sensor_soft_fail_as_skipped
Expand Down
1 change: 1 addition & 0 deletions tests/operators/test_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ def create_dag_run(self) -> DagRun:
session=self.dag_maker.session,
execution_date=self.default_date,
run_type=DagRunType.MANUAL,
data_interval=(self.default_date, self.default_date),
)

def create_ti(self, fn, **kwargs) -> TI:
Expand Down

0 comments on commit 0a98435

Please sign in to comment.