Skip to content

Commit

Permalink
Remove _read method from hdfs task handler after bumping min airflow …
Browse files Browse the repository at this point in the history
…version to 2.6 (#36425)
  • Loading branch information
hussein-awala committed Dec 26, 2023
1 parent af9328e commit dd96be5
Showing 1 changed file with 0 additions and 28 deletions.
28 changes: 0 additions & 28 deletions airflow/providers/apache/hdfs/log/hdfs_task_handler.py
Expand Up @@ -103,31 +103,3 @@ def _read_remote_logs(self, ti, try_number, metadata=None):
messages.append(f"No logs found on hdfs for ti={ti}")

return messages, logs

def _read(self, ti, try_number, metadata=None):
"""
Read logs of given task instance and try_number from HDFS.
If failed, read the log from task instance host machine.
todo: when min airflow version >= 2.6 then remove this method (``_read``)
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
can be used for steaming log reading and auto-tailing.
"""
# from airflow 2.6 we no longer implement the _read method
if hasattr(super(), "_read_remote_logs"):
return super()._read(ti, try_number, metadata)
# if we get here, we're on airflow < 2.6 and we use this backcompat logic
messages, logs = self._read_remote_logs(ti, try_number, metadata)
if logs:
return "".join(f"*** {x}\n" for x in messages) + "\n".join(logs), {"end_of_log": True}
else:
if metadata and metadata.get("log_pos", 0) > 0:
log_prefix = ""
else:
log_prefix = "*** Falling back to local log\n"
local_log, metadata = super()._read(ti, try_number, metadata)
return f"{log_prefix}{local_log}", metadata

0 comments on commit dd96be5

Please sign in to comment.