Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix output property missing for airflow version < 2.4.0 #1385

Merged
merged 14 commits into from
Dec 12, 2022
Merged
33 changes: 27 additions & 6 deletions python-sdk/src/astro/sql/operators/cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,17 +224,38 @@ def resolve_tables_from_tasks( # noqa: C901
for task in tasks:
try:
if isinstance(task, OPERATOR_CLASSES_WITH_TABLE_OUTPUT):
t = task.output.resolve(context)
if isinstance(t, BaseTable):
res.append(t)
try:
# works on airflow version >= 2.4.0
sunank200 marked this conversation as resolved.
Show resolved Hide resolved
t = task.output.resolve(context)
if isinstance(t, BaseTable):
res.append(t)
except AttributeError:
sunank200 marked this conversation as resolved.
Show resolved Hide resolved
# works on airflow version < 2.4.0
from airflow.models.xcom_arg import XComArg

task_output = XComArg(operator=self)
for t in task_output.resolve(context):
if isinstance(t, BaseTable):
res.append(t)
sunank200 marked this conversation as resolved.
Show resolved Hide resolved
elif (
MappedOperator
and isinstance(task, MappedOperator)
and issubclass(task.operator_class, OPERATOR_CLASSES_WITH_TABLE_OUTPUT)
):
for t in task.output.resolve(context):
if isinstance(t, BaseTable):
res.append(t)
try:
# works on airflow version >= 2.4.0
for t in task.output.resolve(context):
if isinstance(t, BaseTable):
res.append(t)
except AttributeError:
# works on airflow version < 2.4.0
from airflow.models.xcom_arg import XComArg

task_output = XComArg(operator=self)
for t in task_output.resolve(context):
if isinstance(t, BaseTable):
res.append(t)

except AirflowException: # pragma: no cover
self.log.info(
"xcom output for %s not found. Will not clean up this task",
Expand Down