From 746f68f759b38a5f73ca9c8cc3aef6385b07be66 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 13 Dec 2022 14:33:14 +0200 Subject: [PATCH 1/4] fixed --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../airbyte_cdk/sources/utils/stream_helpers.py | 11 +++++++---- airbyte-cdk/python/setup.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 1a7357cc95d3c..1c709768dfe2d 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.13.2 +Fixed `StopIteration` exception for empty streams while `check-availability` runs. + ## 0.13.2 Low-code: Enable low-code CDK users to specify schema inline in the manifest diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py index 6f972246693bf..bd922f3d80aa5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py @@ -17,10 +17,13 @@ def get_first_record(self, stream: Stream) -> StreamData: :param stream: stream :return: StreamData containing the first record in the stream """ - # Some streams need a stream slice to read records (e.g. if they have a SubstreamSlicer) - stream_slice = self.get_stream_slice(stream) - records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - next(records) + try: + # Some streams need a stream slice to read records (e.g. if they have a SubstreamSlicer) + stream_slice = self.get_stream_slice(stream) + records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + return next(records) + except StopIteration: + return {} @staticmethod def get_stream_slice(stream: Stream) -> Optional[Mapping[str, Any]]: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 91bf0d7b69eef..ef72c6ae3735c 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.13.2", + version="0.13.3", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 1bdc1cbeae9224086230f84c8bee7ef46b4f6e14 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 13 Dec 2022 09:34:31 -0500 Subject: [PATCH 2/4] Add test for empty stream --- .../http/test_availability_strategy.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 1168dc609ca03..7158d77a5b440 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -139,3 +139,31 @@ def test_send_handles_retries_when_checking_availability(mocker, caplog): assert mock_send.call_count == 3 for message in ["Caught retryable error", "Response Code: 429", "Response Code: 503"]: assert message in caplog.text + + +def test_http_availability_strategy_on_empty_stream(mocker): + mocker.patch.multiple(HttpStream, __abstractmethods__=set()) + mocker.patch.multiple(Stream, __abstractmethods__=set()) + + class MockEmptyStream(mocker.MagicMock, HttpStream): + page_size = None + get_json_schema = mocker.MagicMock() + + def __init__(self, *args, **kvargs): + mocker.MagicMock.__init__(self) + self.read_records = mocker.MagicMock() + + empty_stream = MockEmptyStream() + streams = [empty_stream] + assert isinstance(empty_stream, HttpStream) + + assert isinstance(empty_stream.availability_strategy, HttpAvailabilityStrategy) + + # Generator should have no values to generate + empty_stream.read_records.return_value = iter([]) + + logger = logging.getLogger("airbyte.test-source") + stream_is_available, _ = empty_stream.check_availability(logger) + + assert stream_is_available + assert empty_stream.read_records.called From 53a977df20f60efbbe451fd14c8009117ac0ebc1 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 13 Dec 2022 09:36:55 -0500 Subject: [PATCH 3/4] Correct changelog version --- airbyte-cdk/python/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 1c709768dfe2d..76bb44ebfb9b5 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog -## 0.13.2 -Fixed `StopIteration` exception for empty streams while `check-availability` runs. +## 0.13.3 +Fixed `StopIteration` exception for empty streams while `check_availability` runs. ## 0.13.2 Low-code: Enable low-code CDK users to specify schema inline in the manifest From 2af679207c347d3a364329386130353bc88b7436 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 13 Dec 2022 10:01:07 -0500 Subject: [PATCH 4/4] Fix flake8 issue --- .../sources/streams/http/test_availability_strategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 7158d77a5b440..68c0b7e04dbf0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -154,7 +154,6 @@ def __init__(self, *args, **kvargs): self.read_records = mocker.MagicMock() empty_stream = MockEmptyStream() - streams = [empty_stream] assert isinstance(empty_stream, HttpStream) assert isinstance(empty_stream.availability_strategy, HttpAvailabilityStrategy)