Skip to content

Commit

Permalink
Fix sftp sensor with pattern (#29467)
Browse files Browse the repository at this point in the history
* Solve file_pattern issue with file path creation

This solves #28121

* Correct file_by_pattern.return_value in test

The output of SFTPHook.get_file_by_pattern is a file name and not the
full path to the file.

* Remove test_file_with_pattern_parameter_call

To my understanding, the goal of `test_file_with_pattern_parameter_call`
is to test whether we can call the function with the `file_pattern`
parameter.
Due to the previous changes, the `os.path.join` requires to set a value
for `sftp_hook_mock.return_value.get_file_by_pattern.return_value` so that the
`poke` function would run.

The case where we set this value is dealt with
`test_file_present_with_pattern` and
`test_file_not_present_with_pattern`.
`test_file_with_pattern_parameter_call` thus does not test anything
different or more specific than the two other tests.

---------

Co-authored-by: b.ramsauer@swiss-sdi.ch <b.ramsauer@swiss-sdi.ch>
  • Loading branch information
BenediktRamsauer and BenediktRamsauer committed Feb 13, 2023
1 parent 72c3817 commit 8e24387
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 11 deletions.
3 changes: 2 additions & 1 deletion airflow/providers/sftp/sensors/sftp.py
Expand Up @@ -18,6 +18,7 @@
"""This module contains SFTP sensor."""
from __future__ import annotations

import os
from datetime import datetime
from typing import TYPE_CHECKING, Sequence

Expand Down Expand Up @@ -69,7 +70,7 @@ def poke(self, context: Context) -> bool:
if self.file_pattern:
file_from_pattern = self.hook.get_file_by_pattern(self.path, self.file_pattern)
if file_from_pattern:
actual_file_to_check = file_from_pattern
actual_file_to_check = os.path.join(self.path, file_from_pattern)
else:
return False
else:
Expand Down
11 changes: 1 addition & 10 deletions tests/providers/sftp/sensors/test_sftp.py
Expand Up @@ -98,19 +98,10 @@ def test_naive_datetime(self, sftp_hook_mock):
sftp_hook_mock.return_value.get_mod_time.assert_called_once_with("/path/to/file/1970-01-01.txt")
assert not output

@patch("airflow.providers.sftp.sensors.sftp.SFTPHook")
def test_file_with_pattern_parameter_call(self, sftp_hook_mock):
sftp_hook_mock.return_value.get_mod_time.return_value = "19700101000000"
sftp_sensor = SFTPSensor(task_id="unit_test", path="/path/to/file/", file_pattern="*.txt")
context = {"ds": "1970-01-01"}
output = sftp_sensor.poke(context)
sftp_hook_mock.return_value.get_file_by_pattern.assert_called_once_with("/path/to/file/", "*.txt")
assert output

@patch("airflow.providers.sftp.sensors.sftp.SFTPHook")
def test_file_present_with_pattern(self, sftp_hook_mock):
sftp_hook_mock.return_value.get_mod_time.return_value = "19700101000000"
sftp_hook_mock.return_value.get_file_by_pattern.return_value = "/path/to/file/text_file.txt"
sftp_hook_mock.return_value.get_file_by_pattern.return_value = "text_file.txt"
sftp_sensor = SFTPSensor(task_id="unit_test", path="/path/to/file/", file_pattern="*.txt")
context = {"ds": "1970-01-01"}
output = sftp_sensor.poke(context)
Expand Down

0 comments on commit 8e24387

Please sign in to comment.