From 8570f591870455158ae87f54f63e7fead4867f18 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 23 Jan 2023 15:52:43 -0600 Subject: [PATCH 01/40] Restore HttpAvailabilityStrategy as default (revert https://github.com/airbytehq/airbyte/pull/21488) --- .../airbyte_cdk/sources/streams/http/http.py | 6 ++++++ .../python/docs/concepts/http-streams.md | 17 +++++++++++------ .../declarative/checks/test_check_stream.py | 6 ------ .../streams/http/test_availability_strategy.py | 6 ------ .../python/unit_tests/sources/test_source.py | 11 ----------- 5 files changed, 17 insertions(+), 29 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 084eb052894ed..35cef60807fe9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -13,7 +13,9 @@ import requests import requests_cache from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream, StreamData +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests.auth import AuthBase from requests_cache.session import CachedSession @@ -113,6 +115,10 @@ def retry_factor(self) -> float: def authenticator(self) -> HttpAuthenticator: return self._authenticator + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return HttpAvailabilityStrategy() + @abstractmethod def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ diff --git a/airbyte-cdk/python/docs/concepts/http-streams.md b/airbyte-cdk/python/docs/concepts/http-streams.md index aa80d05fc266d..0008ea281153b 100644 --- a/airbyte-cdk/python/docs/concepts/http-streams.md +++ b/airbyte-cdk/python/docs/concepts/http-streams.md @@ -87,15 +87,20 @@ be returned as a keyword argument. The CDK defines an `AvailabilityStrategy` for a stream, which is used to perform the `check_availability` method. This method checks whether the stream is available before performing `read_records`. -For HTTP streams, a `HttpAvailabilityStrategy` is defined, which attempts to read the first record of the stream, and excepts +For HTTP streams, a default `HttpAvailabilityStrategy` is defined, which attempts to read the first record of the stream, and excepts a dictionary of known error codes and associated reasons, `reasons_for_unavailable_status_codes`. By default, this list contains only `requests.status_codes.FORBIDDEN` (403), with an associated error message that tells the user that they are likely missing permissions associated with that stream. -You can use this `HttpAvailabilityStrategy` in your `HttpStream` by adding the following property to your stream class: +### Customizing stream availability + +You can subclass `HttpAvailabilityStrategy` to override the `reasons_for_unavailable_status_codes` to except more HTTP error codes and inform the user how to resolve errors specific to your connector or stream. + +### Disabling stream availability check + +You can disable the `HttpAvailabilityStrategy` in your `HttpStream` by adding the following property to your stream class: ```python + @property def availability_strategy(self) -> Optional[AvailabilityStrategy]: - return HttpAvailabilityStrategy() -``` - -You can also subclass `HttpAvailabilityStrategy` to override the list of known errors to except more error codes and inform the user how to resolve errors specific to your connector or stream. + return None +``` \ No newline at end of file diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 52a2e4f03500b..2c5670a0896ee 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -9,7 +9,6 @@ import pytest import requests from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy @@ -120,11 +119,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield stub_resp pass - # TODO (Ella): Remove explicit definition when turning on default - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return HttpAvailabilityStrategy() - http_stream = MockHttpStream() assert isinstance(http_stream, HttpStream) assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 6bd1cf005fa26..4afed0fd11886 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -9,7 +9,6 @@ import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.http import HttpStream from requests import HTTPError @@ -40,11 +39,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def retry_factor(self) -> float: return 0.01 - # TODO (Ella): Remove explicit definition when turning on default - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return HttpAvailabilityStrategy() - @pytest.mark.parametrize( ("status_code", "json_contents", "expected_is_available", "expected_messages"), diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index 0cae909b2f083..64a546108db14 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -23,7 +23,6 @@ Type, ) from airbyte_cdk.sources import AbstractSource, Source -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.http import HttpStream @@ -492,11 +491,6 @@ def __init__(self, *args, **kvargs): HttpStream.__init__(self, *args, kvargs) self.read_records = mocker.MagicMock() - # TODO (Ella): Remove explicit definition when turning on default - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return HttpAvailabilityStrategy() - class MockStream(mocker.MagicMock, Stream): page_size = None get_json_schema = mocker.MagicMock() @@ -550,11 +544,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp self.resp_counter += 1 yield stub_response - # TODO (Ella): Remove explicit definition when turning on default - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return HttpAvailabilityStrategy() - class MockStream(mocker.MagicMock, Stream): page_size = None get_json_schema = mocker.MagicMock() From 8dc7b376153c87549f6ffdc22a7aad86024433bb Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:09:00 -0600 Subject: [PATCH 02/40] Turn off default for source-gitlab (default won't work for some streams on this source) --- airbyte-integrations/connectors/source-gitlab/Dockerfile | 2 +- .../connectors/source-gitlab/source_gitlab/streams.py | 5 +++++ docs/integrations/sources/gitlab.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index 5466ec492148d..f3d921dc247b3 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 5d550b7caeb29..facc50a90b04d 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy class GitlabStream(HttpStream, ABC): @@ -41,6 +42,10 @@ def request_params( def url_base(self) -> str: return f"https://{self.api_url}/api/v4/" + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def should_retry(self, response: requests.Response) -> bool: # Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance). if response.status_code == 403: diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index 0fa5220f5121a..2b915fa37c316 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| +| 1.0.1 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.0 | 2022-12-05 | [7506](https://github.com/airbytehq/airbyte/pull/7506) | Add `OAuth2.0` authentication option | | 0.1.12 | 2022-12-15 | [20542](https://github.com/airbytehq/airbyte/pull/20542) | Revert HttpAvailability changes, run on cdk 0.15.0 | | 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | From 9931d65380127fc811f82bbdb4d0c0d17c7b99fa Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:12:26 -0600 Subject: [PATCH 03/40] Turn off default for source-amazon-ads (GA) --- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- .../source-amazon-ads/source_amazon_ads/streams/common.py | 5 +++++ docs/integrations/sources/amazon-ads.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index c886f587e32f2..630c894b5326f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.27 +LABEL io.airbyte.version=0.1.28 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index b955e18ba07dd..3db60973fc090 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -5,6 +5,7 @@ from abc import ABC, abstractmethod from http import HTTPStatus from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from airbyte_cdk.sources.streams import AvailabilityStrategy import requests from airbyte_cdk.sources.streams.core import Stream @@ -90,6 +91,10 @@ def get_json_schema(self): expand_refs(schema) return schema + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + # Basic full refresh stream class AmazonAdsStream(HttpStream, BasicAmazonAdsStream): diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index cf501d1b2bfce..5d6829d667f2a 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,6 +94,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 1.0.28 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.27 | 2023-01-05 | [21082](https://github.com/airbytehq/airbyte/pull/21082) | Fix bug with handling: "Report date is too far in the past." - partial revert of #20662 | | 0.1.26 | 2022-12-19 | [20662](https://github.com/airbytehq/airbyte/pull/20662) | Fix bug with handling: "Report date is too far in the past." | | 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config | From e72fc6ed64bc41930c298f1a32b8f8d152a0d516 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:15:51 -0600 Subject: [PATCH 04/40] Turn off default for source-amplitude (GA) --- airbyte-integrations/connectors/source-amplitude/Dockerfile | 2 +- .../connectors/source-amplitude/source_amplitude/api.py | 5 +++++ docs/integrations/sources/amazon-ads.md | 2 +- docs/integrations/sources/amplitude.md | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile index d1a4466b53053..97c29296b0ecf 100644 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py index 89289c795ec65..6cac8308b4453 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py @@ -16,6 +16,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy from .errors import HTTP_ERROR_CODES, error_msg_from_status @@ -32,6 +33,10 @@ def url_base(self) -> str: subdomain = "analytics.eu." if self.data_region == "EU Residency Server" else "" return f"https://{subdomain}amplitude.com/api/" + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 5d6829d667f2a..6c62dd5bc0b47 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,7 +94,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 1.0.28 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.28 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.27 | 2023-01-05 | [21082](https://github.com/airbytehq/airbyte/pull/21082) | Fix bug with handling: "Report date is too far in the past." - partial revert of #20662 | | 0.1.26 | 2022-12-19 | [20662](https://github.com/airbytehq/airbyte/pull/20662) | Fix bug with handling: "Report date is too far in the past." | | 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config | diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md index 265226a6ed743..ebbc7783bf4ed 100644 --- a/docs/integrations/sources/amplitude.md +++ b/docs/integrations/sources/amplitude.md @@ -43,6 +43,7 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 0.1.20 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.19 | 2022-12-09 | [19727](https://github.com/airbytehq/airbyte/pull/19727) | Remove `data_region` as required | | 0.1.18 | 2022-12-08 | [19727](https://github.com/airbytehq/airbyte/pull/19727) | Add parameter to select region | | 0.1.17 | 2022-10-31 | [18684](https://github.com/airbytehq/airbyte/pull/18684) | Add empty `series` validation for `AverageSessionLength` stream | From 1cda6dcc810db3beaacaf62c60b1ee795bb8fbbc Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:19:43 -0600 Subject: [PATCH 05/40] Turn off default for source-facebook-marketing (GA) --- .../connectors/source-facebook-marketing/Dockerfile | 2 +- .../source_facebook_marketing/streams/base_streams.py | 5 +++++ docs/integrations/sources/facebook-marketing.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 03e57ca90bfa2..79ae0360b5f59 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.83 +LABEL io.airbyte.version=0.2.84 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 7a1a8f628e741..45011220c8677 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -16,6 +16,7 @@ from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.api import FacebookAdsApiBatch, FacebookRequest, FacebookResponse +from airbyte_cdk.sources.streams import AvailabilityStrategy from .common import deep_merge @@ -40,6 +41,10 @@ class FBMarketingStream(Stream, ABC): # entity prefix for `include_deleted` filter, it usually matches singular version of stream name entity_prefix = None + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, max_batch_size: int = 50, **kwargs): super().__init__(**kwargs) self._api = api diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index bd21d47fa2c5e..71582bf16d8e1 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -133,6 +133,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.84 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` || | | | | | 0.2.83 | 2023-01-13 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Videos stream remove filtering | | 0.2.82 | 2023-01-09 | [21149](https://github.com/airbytehq/airbyte/pull/21149) | Fix AdAccount schema | | 0.2.81 | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request | From 532d10520d23a9b2cb79c0963a5b413294df94e9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:22:30 -0600 Subject: [PATCH 06/40] Turn off default for source-freshdesk (GA) --- airbyte-integrations/connectors/source-freshdesk/Dockerfile | 2 +- .../connectors/source-freshdesk/source_freshdesk/streams.py | 5 +++++ docs/integrations/sources/freshdesk.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile index 3de5ddb704242..571345eb17a07 100644 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.0 +LABEL io.airbyte.version=2.0.1 LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 83877e24ab654..6724bec771089 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.streams.core import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.sources.streams import AvailabilityStrategy from requests.auth import AuthBase from source_freshdesk.utils import CallCredit @@ -46,6 +47,10 @@ def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, ** def url_base(self) -> str: return parse.urljoin(f"https://{self.domain.rstrip('/')}", "/api/v2/") + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == requests.codes.too_many_requests: return float(response.headers.get("Retry-After", 0)) diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index 7c88aae5aaa85..0aa44480440c7 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -67,6 +67,7 @@ The Freshdesk connector should not run into Freshdesk API limitations under norm | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------| +| 2.0.1 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 2.0.0 | 2022-12-20 | [20416](https://github.com/airbytehq/airbyte/pull/20416) | Fix `SlaPolicies` stream schema | | 1.0.0 | 2022-11-16 | [19496](https://github.com/airbytehq/airbyte/pull/19496) | Fix `Contacts` stream schema | | 0.3.8 | 2022-11-11 | [19349](https://github.com/airbytehq/airbyte/pull/19349) | Do not rely on response.json() when deciding to retry a request | From ced6aaeddbb63c10bbcac5a590538ddc4458744b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:24:40 -0600 Subject: [PATCH 07/40] Turn off default for source-github (GA) --- airbyte-integrations/connectors/source-github/Dockerfile | 2 +- .../connectors/source-github/source_github/streams.py | 5 +++++ docs/integrations/sources/github.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index f3e74bdc4e468..2971a2abe69c6 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.12 +LABEL io.airbyte.version=0.3.13 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index bee2fc394dc9a..bafe345691cac 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -12,6 +12,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from airbyte_cdk.sources.streams import AvailabilityStrategy from requests.exceptions import HTTPError from .graphql import CursorStorage, QueryReactions, get_query_pull_requests, get_query_reviews @@ -42,6 +43,10 @@ def __init__(self, repositories: List[str], page_size_for_large_streams: int, ** self._session.mount("https://", adapter) self._session.mount("http://", adapter) + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/{self.name}" diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index d4cfe5fd1f5b7..8b519a16c81bb 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -163,7 +163,8 @@ The GitHub connector should not run into GitHub API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.3.12 | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481) | Handle 502 Bad Gateway error with proper log message | +| 0.3.13 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | | +| 0.3.12 | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481) | Handle 502 Bad Gateway error with proper log message | | 0.3.11 | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084) | Raise Error if no organizations or repos are available during read | | 0.3.10 | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523) | Revert changes from 0.3.9 | | 0.3.9 | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978) | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes | From 6417cc0022ba6b250a40e5c36641c61622d1b2a9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:29:06 -0600 Subject: [PATCH 08/40] Turn off default for source-google-analytics-v4 (GA) --- .../connectors/source-google-analytics-v4/Dockerfile | 2 +- .../source_google_analytics_v4/source.py | 5 +++++ .../sources/google-analytics-universal-analytics.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index c22623a6e97a4..913bb4a146d63 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.33 +LABEL io.airbyte.version=0.1.34 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index ce63ca168f943..841b71d10c080 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -19,6 +19,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy from .custom_reports_validator import CustomReportsValidator @@ -116,6 +117,10 @@ def __init__(self, config: MutableMapping): def state_checkpoint_interval(self) -> int: return self.window_in_days + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + @staticmethod def to_datetime_str(date: datetime) -> str: """ diff --git a/docs/integrations/sources/google-analytics-universal-analytics.md b/docs/integrations/sources/google-analytics-universal-analytics.md index 47a3d43b20fc8..9c2990572500e 100644 --- a/docs/integrations/sources/google-analytics-universal-analytics.md +++ b/docs/integrations/sources/google-analytics-universal-analytics.md @@ -159,6 +159,7 @@ Incremental sync is supported only if you add `ga:date` dimension to your custom | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.1.34 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.33 | 2022-12-23 | [20858](https://github.com/airbytehq/airbyte/pull/20858) | Fix check connection | | 0.1.32 | 2022-11-04 | [18965](https://github.com/airbytehq/airbyte/pull/18965) | Fix for `discovery` stage, when `custom_reports` are provided with single stream as `dict` | | 0.1.31 | 2022-10-30 | [18670](https://github.com/airbytehq/airbyte/pull/18670) | Add `Custom Reports` schema validation on `check connection` | From 8434c97666fae1891f528a3a9c33a9b30679fa0b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:31:05 -0600 Subject: [PATCH 09/40] Turn off default for source-google-search-console (GA) --- .../connectors/source-google-search-console/Dockerfile | 2 +- .../source_google_search_console/streams.py | 5 +++++ docs/integrations/sources/google-search-console.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index c48b29620b6f6..ebe4c0af5322d 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index c52d3b0599b26..f5312291da9dc 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -11,6 +11,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy BASE_URL = "https://www.googleapis.com/webmasters/v3/" ROW_LIMIT = 25000 @@ -33,6 +34,10 @@ def __init__( self._start_date = start_date self._end_date = end_date + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + @staticmethod def sanitize_urls_list(site_urls: list) -> List[str]: return list(map(quote_plus, site_urls)) diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md index 6e6011992c794..aba6e18de5574 100644 --- a/docs/integrations/sources/google-search-console.md +++ b/docs/integrations/sources/google-search-console.md @@ -131,7 +131,8 @@ This connector attempts to back off gracefully when it hits Reports API's rate l ## Changelog | Version | Date | Pull Request | Subject | -| :------- | :--------- | :------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------- | +|:---------| :--------- |:--------------------------------------------------------------------------------------------------------------| :---------------------------------------------------------- | +| `0.1.19` | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | `0.1.18` | 2022-10-27 | [18568](https://github.com/airbytehq/airbyte/pull/18568) | Improved config validation: custom_reports.dimension | | `0.1.17` | 2022-10-08 | [17751](https://github.com/airbytehq/airbyte/pull/17751) | Improved config validation: start_date, end_date, site_urls | | `0.1.16` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | From ac5afae1ae1f8c40227161d90360292ce81b9210 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:34:29 -0600 Subject: [PATCH 10/40] Turn off default for source-harvest (GA) --- airbyte-integrations/connectors/source-harvest/Dockerfile | 2 +- .../connectors/source-harvest/source_harvest/streams.py | 5 +++++ docs/integrations/sources/harvest.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile index dc02e76d95456..8e50a0e3feb4e 100644 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index 6179c28a39954..bf41575feda1e 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy class HarvestStream(HttpStream, ABC): @@ -25,6 +26,10 @@ def data_field(self) -> str: """ return self.name + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def backoff_time(self, response: requests.Response): if "Retry-After" in response.headers: return int(response.headers["Retry-After"]) diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index e0a2c67e4ed84..089c1bebcbe9f 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -79,6 +79,7 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------| +| 0.1.25 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` || | | | | | 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | | 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | | 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | From 317ada1ef3e5212800e58fb9d0c723092b8fb5e3 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:36:43 -0600 Subject: [PATCH 11/40] Turn off default for source-hubspot (GA) --- airbyte-integrations/connectors/source-hubspot/Dockerfile | 2 +- .../connectors/source-hubspot/source_hubspot/streams.py | 5 +++++ docs/integrations/sources/hubspot.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 28c1d2e539c78..634c0a82b7ad4 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index da0d30f13f2f3..4a017bc45759c 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -17,6 +17,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests import codes from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS @@ -209,6 +210,10 @@ class Stream(HttpStream, ABC): granted_scopes: Set = None properties_scopes: Set = None + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + @property @abstractmethod def scopes(self) -> Set[str]: diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index b08234edb147f..c4417e2341051 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -126,6 +126,7 @@ Now that you have set up the Hubspot source connector, check out the following H | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.1 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.3.0 | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available | | 0.2.2 | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation | | 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | From fb1bd65fdbe255a85f840a21143f037ec37a8771 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:39:10 -0600 Subject: [PATCH 12/40] Turn off default for source-intercom (GA) --- airbyte-integrations/connectors/source-intercom/Dockerfile | 2 +- .../connectors/source-intercom/source_intercom/source.py | 5 +++++ docs/integrations/sources/intercom.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 6377255906f5c..6f258a0de2ebe 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -35,5 +35,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.29 +LABEL io.airbyte.version=0.1.30 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 161c89ff3bfd3..95889ac2a9c38 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy from requests.auth import AuthBase from .utils import EagerlyCachedStreamState as stream_state_cache @@ -42,6 +43,10 @@ def authenticator(self): return self._session.auth return super().authenticator + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response, **kwargs) -> Optional[Mapping[str, Any]]: """ Abstract method of HttpStream - should be overwritten. diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index ed93742ea99d2..1dff687f80717 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -48,7 +48,8 @@ The Intercom connector should not run into Intercom API limitations under normal ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +| 0.1.30 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.29 | 2022-10-31 | [18681](https://github.com/airbytehq/airbyte/pull/18681) | Define correct version for airbyte-cdk~=0.2 | | 0.1.28 | 2022-10-20 | [18216](https://github.com/airbytehq/airbyte/pull/18216) | Use airbyte-cdk~=0.2.0 with SQLite caching | | 0.1.27 | 2022-08-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | From 959e57765c0343d3cbc6ad5a5853c7e7e3c71432 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:40:31 -0600 Subject: [PATCH 13/40] Turn off default for source-iterable (GA) --- airbyte-integrations/connectors/source-iterable/Dockerfile | 2 +- .../connectors/source-iterable/source_iterable/streams.py | 5 +++++ docs/integrations/sources/iterable.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index 889e9f37f0e43..7afa380648c1b 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.22 +LABEL io.airbyte.version=0.1.23 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index 2640a6ae3e140..b2bc6e83bd05b 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -15,6 +15,7 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.streams import AvailabilityStrategy from pendulum.datetime import DateTime from requests import codes from requests.exceptions import ChunkedEncodingError @@ -48,6 +49,10 @@ def data_field(self) -> str: :return: Default field name to get data from response """ + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def check_unauthorized_key(self, response: requests.Response) -> bool: if response.status_code == codes.UNAUTHORIZED: self.logger.warn(f"Provided API Key has not sufficient permissions to read from stream: {self.data_field}") diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index 429e45b235026..0312a98113cde 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -76,6 +76,7 @@ The Iterable source connector supports the following [sync modes](https://docs.a | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.23 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.22 | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak | | 0.1.21 | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery | | 0.1.20 | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors | From 37eef69ed6ff630c09ddc55830c93d3118317287 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:42:32 -0600 Subject: [PATCH 14/40] Turn off default for source-klaviyo (GA) --- airbyte-integrations/connectors/source-klaviyo/Dockerfile | 2 +- .../connectors/source-klaviyo/source_klaviyo/streams.py | 5 +++++ docs/integrations/sources/klaviyo.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile index 4e769f06f7328..2d85dbc952e7f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ b/airbyte-integrations/connectors/source-klaviyo/Dockerfile @@ -34,5 +34,5 @@ COPY source_klaviyo ./source_klaviyo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index abf59dbd31e60..5bb834ad4813c 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.sources.streams import AvailabilityStrategy class KlaviyoStream(HttpStream, ABC): @@ -27,6 +28,10 @@ def __init__(self, api_key: str, **kwargs): transform_function = self.get_custom_transform() self.transformer.registerCustomTransform(transform_function) + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def get_custom_transform(self): def custom_transform_date_rfc3339(original_value, field_schema): if original_value and "format" in field_schema and field_schema["format"] == "date-time": diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md index dc8ed649f0720..bab247c81b0e9 100644 --- a/docs/integrations/sources/klaviyo.md +++ b/docs/integrations/sources/klaviyo.md @@ -54,6 +54,7 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u | Version | Date | Pull Request | Subject | |:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------| +| `0.1.11` | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency | | `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state. | | `0.1.6` | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage | From c9a4d31d16ace08c1b04ee34bec5ee3bdfebde49 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:46:16 -0600 Subject: [PATCH 15/40] Turn off default for source-linkedin-ads (GA) --- .../connectors/source-linkedin-ads/Dockerfile | 2 +- .../source-linkedin-ads/source_linkedin_ads/source.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index d0bfa957d1b10..65e06ab441614 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index a4a3bcb5a692f..778aee21f9525 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -17,6 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from airbyte_cdk.sources.streams import AvailabilityStrategy from .analytics import make_analytics_slices, merge_chunks, update_analytics_params from .utils import get_parent_stream_values, transform_data @@ -42,6 +43,10 @@ def accounts(self): """Property to return the list of the user Account Ids from input""" return ",".join(map(str, self.config.get("account_ids"))) + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path(self, **kwargs) -> str: """Returns the API endpoint path for stream, from `endpoint` class attribute.""" return self.endpoint From e061a0c5bed654d3bb5fcc98251e36bfdee86dc0 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:49:02 -0600 Subject: [PATCH 16/40] Turn off default for source-mailchimp (GA) --- airbyte-integrations/connectors/source-mailchimp/Dockerfile | 2 +- .../connectors/source-mailchimp/source_mailchimp/streams.py | 5 +++++ docs/integrations/sources/mailchimp.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile index 6789e0b3806fb..c9ce055012d51 100644 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile +++ b/airbyte-integrations/connectors/source-mailchimp/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 40b42330d5a7f..21e6463d697b3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -6,6 +6,7 @@ import math from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from airbyte_cdk.sources.streams import AvailabilityStrategy import requests from airbyte_cdk.models import SyncMode @@ -25,6 +26,10 @@ def __init__(self, **kwargs): def url_base(self) -> str: return f"https://{self.data_center}.api.mailchimp.com/3.0/" + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() api_data = decoded_response[self.data_field] diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index a366a6d9e52a3..9795761e61b40 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -230,6 +230,7 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| +| 0.3.2 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.3.1 | 2022-12-20 | [20720](https://github.com/airbytehq/airbyte/pull/20720) | Use stream slices as a source for request params instead of a stream state | | 0.3.0 | 2022-11-07 | [19023](https://github.com/airbytehq/airbyte/pull/19023) | Set primary key for Email Activity stream. | | 0.2.15 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | From 50d0b0a8301ada651476dc86146f64a6d9f46cb9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:50:30 -0600 Subject: [PATCH 17/40] Turn off default for source-marketo (GA) --- airbyte-integrations/connectors/source-marketo/Dockerfile | 2 +- .../connectors/source-marketo/source_marketo/source.py | 5 +++++ docs/integrations/sources/marketo.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index 2535bd7ba33de..7debb6366db5c 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index aef902242d439..9a95a4d546e41 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -16,6 +16,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy from .utils import STRING_TYPES, clean_string, format_value, to_datetime_str @@ -41,6 +42,10 @@ def __init__(self, config: Mapping[str, Any], stream_name: str = None, param: Ma def url_base(self) -> str: return self._url_base + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path(self, **kwargs) -> str: return f"rest/v1/{self.name}.json" diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index 85c544b2497a2..d1abfb2da3efa 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -106,6 +106,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| `0.1.12` | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | `0.1.11` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Do not use temporary files for memory optimization | | `0.1.10` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Optimize memory consumption | | `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream sate. | From 778da43b3c33bae076d2f2473ce360711de40b61 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:52:03 -0600 Subject: [PATCH 18/40] Turn off default for source-mixpanel (GA) --- airbyte-integrations/connectors/source-mixpanel/Dockerfile | 2 +- .../source-mixpanel/source_mixpanel/streams/base.py | 5 +++++ docs/integrations/sources/mixpanel.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index cd44abfc08fe8..4670149deea78 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.29 +LABEL io.airbyte.version=0.1.30 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 2ddd6af68a1dc..a73135ba5878b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -11,6 +11,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy from pendulum import Date @@ -55,6 +56,10 @@ def __init__( super().__init__(authenticator=authenticator) + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """Define abstract method""" return None diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index df8be46d181ef..0e21b66dd00aa 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -49,7 +49,8 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits ## CHANGELOG | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- | +| 0.1.30 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.29 | 2022-11-02 | [18846](https://github.com/airbytehq/airbyte/pull/18846) | For "export" stream make line parsing more robust | | 0.1.28 | 2022-10-06 | [17699](https://github.com/airbytehq/airbyte/pull/17699) | Fix discover step issue cursor field None | | 0.1.27 | 2022-09-29 | [17415](https://github.com/airbytehq/airbyte/pull/17415) | Disable stream "cohort_members" on discover if not access | From d1045790518076a54dffc593f74673d62e9007b4 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:54:48 -0600 Subject: [PATCH 19/40] Turn off default for source-notion (GA) --- airbyte-integrations/connectors/source-notion/Dockerfile | 2 +- .../connectors/source-notion/source_notion/streams.py | 5 +++++ docs/integrations/sources/notion.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index d910782187e81..c6dad22d2b0b5 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index db2671ca2005e..ddec83c976847 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -9,6 +9,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams import AvailabilityStrategy from .utils import transform_properties @@ -28,6 +29,10 @@ def __init__(self, config: Mapping[str, Any], **kwargs): super().__init__(**kwargs) self.start_date = config["start_date"] + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def backoff_time(self, response: requests.Response) -> Optional[float]: retry_after = response.headers.get("retry-after") if retry_after: diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 076358f38e69d..8e3822f97e66b 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -84,6 +84,7 @@ The connector is restricted by Notion [request limits](https://developers.notion | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------| +| 1.0.1 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | | 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | | 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | From 35b873fcbbcde6ea882d8b646a3a51c3ed377137 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:56:41 -0600 Subject: [PATCH 20/40] Turn off default for source-paypal-transaction (GA) --- .../connectors/source-paypal-transaction/Dockerfile | 2 +- .../source_paypal_transaction/source.py | 5 +++++ docs/integrations/sources/paypal-transaction.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile index 9e26b7a545bcf..fea16324637fe 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 9606eb2de99df..976958c72b779 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -17,6 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.sources.streams import AvailabilityStrategy from dateutil.parser import isoparse from .utils import middle_date_slices @@ -137,6 +138,10 @@ def validate_input_dates(self): def url_base(self) -> str: return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def request_headers(self, **kwargs) -> Mapping[str, Any]: return {"Content-Type": "application/json"} diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index ef7d62962f917..82ed3c893588d 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -87,6 +87,7 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------| +| 0.1.11 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.10 | 2022-09-04 | [17554](https://github.com/airbytehq/airbyte/pull/17554) | Made the spec and source config to be consistent | | 0.1.9 | 2022-08-18 | [15741](https://github.com/airbytehq/airbyte/pull/15741) | Removed `OAuth2.0` option | | 0.1.8 | 2022-07-25 | [15000](https://github.com/airbytehq/airbyte/pull/15000) | Added support of `OAuth2.0` authentication, fixed bug when normalization couldn't handle nested cursor field and primary key | From a3642dd9442a78cba604de04e5e0c032b75bfca3 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:58:24 -0600 Subject: [PATCH 21/40] Turn off default for source-pinterest (GA) --- airbyte-integrations/connectors/source-pinterest/Dockerfile | 2 +- .../connectors/source-pinterest/source_pinterest/source.py | 5 +++++ docs/integrations/sources/pinterest.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile index 1f4efba5560f9..75343735ce333 100644 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ b/airbyte-integrations/connectors/source-pinterest/Dockerfile @@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 80688424bb52d..5af5eff79fe85 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @@ -44,6 +45,10 @@ def start_date(self): def window_in_days(self): return 30 # Set window_in_days to 30 days date range + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: next_page = response.json().get("bookmark", {}) if self.data_fields else {} diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index ab9348e25854b..b27a03209d25c 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -70,7 +70,8 @@ The connector is restricted by the Pinterest [requests limitation](https://devel ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | +|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | +| 0.2.2 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version| | 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Added data-type normalization up to the schemas declared | | 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Added filter based on statuses | From 9365731cf53c814a5fd9986f54642715ea34fb7a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 16:59:54 -0600 Subject: [PATCH 22/40] Turn off default for source-recharge (GA) --- airbyte-integrations/connectors/source-recharge/Dockerfile | 2 +- .../connectors/source-recharge/source_recharge/api.py | 5 +++++ docs/integrations/sources/recharge.md | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 69848f6bd9390..bc55a801d19bf 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 73627d359a7f7..1f40707270462 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.sources.streams import AvailabilityStrategy class RechargeStream(HttpStream, ABC): @@ -28,6 +29,10 @@ class RechargeStream(HttpStream, ABC): def data_path(self): return self.name + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 609aff02de69b..6fcb6e3418f9e 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -75,7 +75,8 @@ The Recharge connector should gracefully handle Recharge API limitations under n ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------- | +| 0.2.5 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.2.4 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | | 0.2.3 | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry` | | 0.2.2 | 2022-10-05 | [17608](https://github.com/airbytehq/airbyte/pull/17608) | Skip stream if we receive 403 error | From 24dbdb395931b9a995639567fc826008b87c6a22 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:03:03 -0600 Subject: [PATCH 23/40] Turn off default for source-salesforce (GA) --- airbyte-integrations/connectors/source-salesforce/Dockerfile | 2 +- .../source-salesforce/source_salesforce/streams.py | 5 +++++ docs/integrations/sources/salesforce.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 6a43255b7a187..1080058850936 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.29 +LABEL io.airbyte.version=1.0.30 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index fb5df8ee196e4..e00c0c7bab7bb 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -17,6 +17,7 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from numpy import nan from pendulum import DateTime # type: ignore[attr-defined] @@ -60,6 +61,10 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: def url_base(self) -> str: return self.sf_api.instance_url + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path(self, next_page_token: Mapping[str, Any] = None, **kwargs: Any) -> str: if next_page_token: """ diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index b8d7a4e0a344b..e66551926a062 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -129,6 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| +| 0.1.30 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.29 | 2023-01-05 | [20886](https://github.com/airbytehq/airbyte/pull/20886) | Remove `ActivityMetric` stream | | 1.0.28 | 2022-12-29 | [20927](https://github.com/airbytehq/airbyte/pull/20927) | Fix tests; add expected records | | 1.0.27 | 2022-11-29 | [19869](https://github.com/airbytehq/airbyte/pull/19869) | Remove `AccountHistory` from unsupported BULK streams | From 562618053e920ae52cec6c97279274d146c3d832 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:04:46 -0600 Subject: [PATCH 24/40] Turn off default for source-sentry (GA) --- airbyte-integrations/connectors/source-sentry/Dockerfile | 2 +- .../connectors/source-sentry/source_sentry/streams.py | 5 +++++ docs/integrations/sources/sentry.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index c5f4badb669bb..5d05b54b34075 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 7aea30c3144a1..454154fed32e7 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy class SentryStream(HttpStream, ABC): @@ -25,6 +26,10 @@ def __init__(self, hostname: str, **kwargs): def url_base(self) -> str: return self._url_base + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index b6e59e96121b3..3a5a5f9ac4e5e 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -45,6 +45,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------| +| 0.1.9 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.8 | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync | | 0.1.7 | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states | | 0.1.6 | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK | From 2a5e88a751080f99d1b90f1cfc268b154b27d3a7 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:07:56 -0600 Subject: [PATCH 25/40] Turn off default for source-slack (GA) --- airbyte-integrations/connectors/source-slack/Dockerfile | 2 +- .../connectors/source-slack/source_slack/source.py | 5 +++++ docs/integrations/sources/slack.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile index ac5ad0c8a5086..9cdc57d0838a4 100644 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ b/airbyte-integrations/connectors/source-slack/Dockerfile @@ -17,5 +17,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-slack diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 214db26f27172..2c1067f848512 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -13,6 +13,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from pendulum import DateTime, Period @@ -27,6 +28,10 @@ def max_retries(self) -> int: # Slack's rate limiting can be unpredictable so we increase the max number of retries by a lot before failing return 20 + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """Slack uses a cursor-based pagination strategy. Extract the cursor from the response if it exists and return it in a format diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 9741d94178c6b..af24a7c39c789 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -136,6 +136,7 @@ It is recommended to sync required channels only, this can be done by specifying | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| +| 0.1.22 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.21 | 2023-01-12 | [21321](https://github.com/airbytehq/airbyte/pull/21321) | Retry Timeout error | | 0.1.20 | 2022-12-21 | [20767](https://github.com/airbytehq/airbyte/pull/20767) | Update schema | | 0.1.19 | 2022-12-01 | [19970](https://github.com/airbytehq/airbyte/pull/19970) | Remove OAuth2.0 broken `refresh_token` support | From 1fbf6b73bee2be94031c73ceef881abe4cde475c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:08:18 -0600 Subject: [PATCH 26/40] Turn off default for source-snapchat-marketing (GA) --- .../connectors/source-snapchat-marketing/Dockerfile | 2 +- .../source_snapchat_marketing/source.py | 5 +++++ docs/integrations/sources/snapchat-marketing.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile index 110cc948831cd..abcaa3bf6ddfc 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile @@ -25,5 +25,5 @@ COPY source_snapchat_marketing ./source_snapchat_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py index 4d68e90c5ad20..c5569c4d8a97a 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py @@ -19,6 +19,7 @@ from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.streams import AvailabilityStrategy # https://marketingapi.snapchat.com/docs/#core-metrics # https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities @@ -183,6 +184,10 @@ def response_root_name(self): """Using the class name in lower to set the root node for response parsing""" return self.name + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + @property def response_item_name(self): """Remove last 's' from response_root_name, see example in parse_response function""" diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index 704834f580e2e..36ef06ff746d1 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -113,6 +113,7 @@ Snapchat Marketing API has limitations to 1000 items per page. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------| +| 0.1.13 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.12 | 2023-01-11 | [21267](https://github.com/airbytehq/airbyte/pull/21267) | Fix parse empty error response | | 0.1.11 | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error | | 0.1.10 | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0 | From e63bb0e2d7bea237da279cfd50803793e8a531ff Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:09:46 -0600 Subject: [PATCH 27/40] Turn off default for source-stripe (GA) --- airbyte-integrations/connectors/source-stripe/Dockerfile | 2 +- .../connectors/source-stripe/source_stripe/streams.py | 5 +++++ docs/integrations/sources/stripe.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 9578c23dcd7b0..d9d9305b97704 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.40 +LABEL io.airbyte.version=0.1.41 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index d1947ef1310d5..e2df251270177 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -11,6 +11,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy STRIPE_ERROR_CODES: List = [ # stream requires additional permissions @@ -31,6 +32,10 @@ def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_ self.start_date = start_date self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() if bool(decoded_response.get("has_more", "False")) and decoded_response.get("data", []): diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index ce37f6e0a9269..860e162277602 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -82,6 +82,7 @@ The Stripe connector should not run into Stripe API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.41 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `Payment Intents` stream schema | | 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | | 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | From 2242ee7238303723f12dd24657684af51d151a97 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:11:04 -0600 Subject: [PATCH 28/40] Turn off default for source-surveymonkey (GA) --- .../connectors/source-surveymonkey/Dockerfile | 2 +- .../source-surveymonkey/source_surveymonkey/streams.py | 6 ++++++ docs/integrations/sources/surveymonkey.md | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile index 33fefc2aa6219..18402e2169a4f 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile +++ b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-surveymonkey diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 695cb2b1a236f..915885af51339 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -6,6 +6,8 @@ import urllib.parse from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from airbyte_cdk.sources.streams import AvailabilityStrategy + import pendulum import requests @@ -27,6 +29,10 @@ def __init__(self, start_date: pendulum.datetime, survey_ids: List[str], **kwarg self._start_date = start_date self._survey_ids = survey_ids + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: resp_json = response.json() links = resp_json.get("links", {}) diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index 1bbc4dee8d0e9..ddf67ca92e05f 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -64,7 +64,8 @@ To cover more data from this source we use caching. ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------- | +| 0.1.14 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.13 | 2022-11-29 | [19868](https://github.com/airbytehq/airbyte/pull/19868) | Fix OAuth flow urls | | 0.1.12 | 2022-10-13 | [17964](https://github.com/airbytehq/airbyte/pull/17964) | Add OAuth for Eu and Ca | | 0.1.11 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | From 41f85192fc556b3745552f41cdcde289bf525d62 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:12:51 -0600 Subject: [PATCH 29/40] Turn off default for source-tiktok-marketing (GA) --- .../connectors/source-tiktok-marketing/Dockerfile | 2 +- .../source_tiktok_marketing/streams.py | 5 +++++ docs/integrations/sources/tiktok-marketing.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 2e453506445a0..fa5d80064b90a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 855658f596f40..2602e37125a2c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -18,6 +18,7 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer # TikTok Initial release date is September 2016 @@ -153,6 +154,10 @@ def __init__(self, **kwargs): # only sandbox has non-empty self._advertiser_id self.is_sandbox = bool(self._advertiser_id) + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """All responses have the similar structure: { diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index e648e19e30101..f52a0dd2c155f 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -550,6 +550,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| 1.0.2 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | | 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. | | 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | From c3a03f1c7f82a22cf0b38144046efd2de63e29ed Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:14:26 -0600 Subject: [PATCH 30/40] Turn off default for source-twilio (GA) --- airbyte-integrations/connectors/source-twilio/Dockerfile | 2 +- .../connectors/source-twilio/source_twilio/streams.py | 6 ++++++ docs/integrations/sources/twilio.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile index bcdfcaf5d9200..b189ece2d760f 100644 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 390794f2dee47..5b209423619b2 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -14,6 +14,8 @@ from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator +from airbyte_cdk.sources.streams import AvailabilityStrategy + from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from pendulum.datetime import DateTime from requests.auth import AuthBase @@ -40,6 +42,10 @@ def changeable_fields(self): """ return [] + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def path(self, **kwargs): return f"{self.name.title()}.json" diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 28621040cdfa8..135a2488dfc6e 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -83,6 +83,7 @@ For more information, see [the Twilio docs for rate limitations](https://support | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| +| 0.1.15 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.14 | 2022-11-16 | [19479](https://github.com/airbytehq/airbyte/pull/19479) | Fix date range slicing | | 0.1.13 | 2022-10-25 | [18423](https://github.com/airbytehq/airbyte/pull/18423) | Implement datetime slicing for streams supporting incremental syncs | | 0.1.11 | 2022-09-30 | [17478](https://github.com/airbytehq/airbyte/pull/17478) | Add lookback_window parameters | From 42130a5750b21fe742868fcba61cfaa7fee0ac88 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:16:01 -0600 Subject: [PATCH 31/40] Turn off default for source-zendesk-chat (GA) --- .../connectors/source-zendesk-chat/Dockerfile | 2 +- .../source-zendesk-chat/source_zendesk_chat/streams.py | 6 ++++++ docs/integrations/sources/zendesk-chat.md | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile index 409f1b8a8038a..2cc19b5d9f083 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index 5e26e8119972a..7b00470a65f45 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -10,6 +10,8 @@ import pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy + class Stream(HttpStream, ABC): @@ -20,6 +22,10 @@ class Stream(HttpStream, ABC): limit = 100 + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def request_kwargs( self, stream_state: Mapping[str, Any], diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 36ef03aa98fd6..35a282ad43c70 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -77,7 +77,8 @@ The connector is restricted by Zendesk's [requests limitation](https://developer ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +| 0.1.12 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.11 | 2022-10-18 | [17745](https://github.com/airbytehq/airbyte/pull/17745) | Add Engagements Stream and fix infity looping | | 0.1.10 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.1.9 | 2022-08-23 | [15879](https://github.com/airbytehq/airbyte/pull/15879) | Corrected specification and stream schemas to support backward capability | From 91c14a78ad6b7f2fb8d6665d17836f6db2f8dbe9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:17:26 -0600 Subject: [PATCH 32/40] Turn off default for source-zendesk-support (GA) --- .../connectors/source-zendesk-support/Dockerfile | 2 +- .../source_zendesk_support/streams.py | 6 ++++++ docs/integrations/sources/zendesk-support.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index e6cd2ad493b80..4010966ca4e23 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.20 +LABEL io.airbyte.version=0.2.21 LABEL io.airbyte.name=airbyte/source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 34329111efadd..2ab3771512199 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -28,6 +28,8 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests.auth import AuthBase from requests_futures.sessions import PICKLE_ERROR, FuturesSession +from airbyte_cdk.sources.streams import AvailabilityStrategy + DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY: str = "_last_end_time" @@ -116,6 +118,10 @@ def __init__(self, subdomain: str, start_date: str, ignore_pagination: bool = Fa self._subdomain = subdomain self._ignore_pagination = ignore_pagination + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def backoff_time(self, response: requests.Response) -> Union[int, float]: """ The rate limit is 700 requests per minute diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 98e57025c7363..4d700d2f6a04b 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -60,6 +60,7 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `0.2.21` | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | `0.2.20` | 2022-12-28 | [20900](https://github.com/airbytehq/airbyte/pull/20900) | Remove synchronous time.sleep, add logging, reduce backoff time | | `0.2.19` | 2022-12-09 | [19967](https://github.com/airbytehq/airbyte/pull/19967) | Fix reading response for more than 100k records | | `0.2.18` | 2022-11-29 | [19432](https://github.com/airbytehq/airbyte/pull/19432) | Revert changes from version 0.2.15, use a test read instead | From 4431df65fdadeced09d8441acd20eef87a1f417f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Jan 2023 17:18:40 -0600 Subject: [PATCH 33/40] Turn off default for source-zendesk-talk (GA) --- .../connectors/source-zendesk-talk/Dockerfile | 2 +- .../source-zendesk-talk/source_zendesk_talk/streams.py | 6 ++++++ docs/integrations/sources/zendesk-talk.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile index 2e633bb8fdfbd..a1a523d5025b0 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py index 14b6f25e5d3ae..cae61c104b59a 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py @@ -11,6 +11,8 @@ import pendulum as pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import AvailabilityStrategy + class ZendeskTalkStream(HttpStream, ABC): @@ -33,6 +35,10 @@ def url_base(self) -> str: """API base url based on configured subdomain""" return f"https://{self._subdomain}.zendesk.com/api/v2/channels/voice/" + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + def backoff_time(self, response: requests.Response) -> Optional[float]: """ Override this method to dynamically determine backoff time e.g: by reading the X-Retry-After header. diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md index 773df66a45fde..04c1dd582ad08 100644 --- a/docs/integrations/sources/zendesk-talk.md +++ b/docs/integrations/sources/zendesk-talk.md @@ -76,6 +76,7 @@ The Zendesk connector should not run into Zendesk API limitations under normal u | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- |:----------------------------------| +| `0.1.6` | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | `0.1.5` | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version | | `0.1.4` | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | | `0.1.3` | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | From 953d7d30e7a9a970ac10b2ecd63a9f325fd03b8e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 08:47:54 -0600 Subject: [PATCH 34/40] use correct import --- .../source-amazon-ads/source_amazon_ads/streams/common.py | 2 +- .../connectors/source-amplitude/source_amplitude/api.py | 2 +- .../source_facebook_marketing/streams/base_streams.py | 2 +- .../connectors/source-freshdesk/source_freshdesk/streams.py | 2 +- .../connectors/source-github/source_github/streams.py | 2 +- .../connectors/source-gitlab/source_gitlab/streams.py | 2 +- .../source_google_analytics_v4/source.py | 2 +- .../source_google_search_console/streams.py | 2 +- .../connectors/source-harvest/source_harvest/streams.py | 2 +- .../connectors/source-hubspot/source_hubspot/streams.py | 2 +- .../connectors/source-intercom/source_intercom/source.py | 2 +- .../connectors/source-iterable/source_iterable/streams.py | 2 +- .../connectors/source-klaviyo/source_klaviyo/streams.py | 2 +- .../source-linkedin-ads/source_linkedin_ads/source.py | 2 +- .../connectors/source-mailchimp/source_mailchimp/streams.py | 2 +- .../connectors/source-marketo/source_marketo/source.py | 2 +- .../connectors/source-mixpanel/source_mixpanel/streams/base.py | 2 +- .../connectors/source-notion/source_notion/streams.py | 2 +- .../source_paypal_transaction/source.py | 2 +- .../connectors/source-pinterest/source_pinterest/source.py | 2 +- .../connectors/source-recharge/source_recharge/api.py | 2 +- .../connectors/source-salesforce/source_salesforce/streams.py | 2 +- .../connectors/source-sentry/source_sentry/streams.py | 2 +- .../connectors/source-slack/source_slack/source.py | 2 +- .../source_snapchat_marketing/source.py | 2 +- .../connectors/source-stripe/source_stripe/streams.py | 2 +- .../source-surveymonkey/source_surveymonkey/streams.py | 2 +- .../source-tiktok-marketing/source_tiktok_marketing/streams.py | 2 +- .../connectors/source-twilio/source_twilio/streams.py | 2 +- .../source-zendesk-chat/source_zendesk_chat/streams.py | 2 +- .../source-zendesk-support/source_zendesk_support/streams.py | 2 +- .../source-zendesk-talk/source_zendesk_talk/streams.py | 2 +- 32 files changed, 32 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 3db60973fc090..1a17e3362df0a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from http import HTTPStatus from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy import requests from airbyte_cdk.sources.streams.core import Stream diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py index 6cac8308b4453..4a07ad0f07a05 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py @@ -16,7 +16,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .errors import HTTP_ERROR_CODES, error_msg_from_status diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 45011220c8677..fca5b0297955a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -16,7 +16,7 @@ from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.api import FacebookAdsApiBatch, FacebookRequest, FacebookResponse -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .common import deep_merge diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 6724bec771089..34b0d896668ce 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -14,7 +14,7 @@ from airbyte_cdk.sources.streams.core import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.auth import AuthBase from source_freshdesk.utils import CallCredit diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index bafe345691cac..108f836ac8722 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -12,7 +12,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.exceptions import HTTPError from .graphql import CursorStorage, QueryReactions, get_query_pull_requests, get_query_reviews diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index facc50a90b04d..8524afc9d7902 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class GitlabStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index 841b71d10c080..583bc14d401e2 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -19,7 +19,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .custom_reports_validator import CustomReportsValidator diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index f5312291da9dc..82fc9fab4fa2b 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -11,7 +11,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy BASE_URL = "https://www.googleapis.com/webmasters/v3/" ROW_LIMIT = 25000 diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index bf41575feda1e..6210db91dc4d6 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class HarvestStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 4a017bc45759c..8b58eaf542c3d 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -17,7 +17,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests import codes from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 95889ac2a9c38..299890b1e764c 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -14,7 +14,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.auth import AuthBase from .utils import EagerlyCachedStreamState as stream_state_cache diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index b2bc6e83bd05b..a57db43cc3266 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -15,7 +15,7 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from pendulum.datetime import DateTime from requests import codes from requests.exceptions import ChunkedEncodingError diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 5bb834ad4813c..06960f2ba7c3e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class KlaviyoStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 778aee21f9525..77955917c3d2f 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -17,7 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .analytics import make_analytics_slices, merge_chunks, update_analytics_params from .utils import get_parent_stream_values, transform_data diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 21e6463d697b3..63c569b0eea69 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -6,7 +6,7 @@ import math from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy import requests from airbyte_cdk.models import SyncMode diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 9a95a4d546e41..621f3f66f7be0 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -16,7 +16,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .utils import STRING_TYPES, clean_string, format_value, to_datetime_str diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index a73135ba5878b..2e839742f79d8 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -11,7 +11,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from pendulum import Date diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index ddec83c976847..6dc6e29949d5e 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -9,7 +9,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .utils import transform_properties diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 976958c72b779..cfb35a8d61ec3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -17,7 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from dateutil.parser import isoparse from .utils import middle_date_slices diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 5af5eff79fe85..c74088fd2be32 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -14,7 +14,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 1f40707270462..56f00b84d3cd3 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class RechargeStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index e00c0c7bab7bb..70c11eb9ab944 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -17,7 +17,7 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from numpy import nan from pendulum import DateTime # type: ignore[attr-defined] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 454154fed32e7..2a45bd9de59d8 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -10,7 +10,7 @@ import requests from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class SentryStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 2c1067f848512..e58090ffe498f 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -11,9 +11,9 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource +from airbte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from pendulum import DateTime, Period diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py index c5569c4d8a97a..26a92c7c328f0 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py @@ -19,7 +19,7 @@ from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy # https://marketingapi.snapchat.com/docs/#core-metrics # https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index e2df251270177..38499ef1ab727 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -11,7 +11,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy STRIPE_ERROR_CODES: List = [ # stream requires additional permissions diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 915885af51339..372e65b47a422 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -6,7 +6,7 @@ import urllib.parse from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy import pendulum diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 2602e37125a2c..8400798158049 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -18,7 +18,7 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer # TikTok Initial release date is September 2016 diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 5b209423619b2..0b1c7922a55ef 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -14,7 +14,7 @@ from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from pendulum.datetime import DateTime diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index 7b00470a65f45..c4de25b49b8f7 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -10,7 +10,7 @@ import pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 2ab3771512199..70cdd2654a9ce 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -28,7 +28,7 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests.auth import AuthBase from requests_futures.sessions import PICKLE_ERROR, FuturesSession -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py index cae61c104b59a..c458b2a133d0d 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py @@ -11,7 +11,7 @@ import pendulum as pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import AvailabilityStrategy +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy From 4e84e611ba40f41144d3b15616c9f1a3a308ae2a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 10:06:32 -0600 Subject: [PATCH 35/40] run format on sources --- .../source-amazon-ads/source_amazon_ads/streams/common.py | 2 +- .../connectors/source-amplitude/source_amplitude/api.py | 2 +- .../source_facebook_marketing/streams/base_streams.py | 2 +- .../connectors/source-freshdesk/source_freshdesk/streams.py | 2 +- .../connectors/source-github/source_github/streams.py | 2 +- .../source_google_analytics_v4/source.py | 2 +- .../source_google_search_console/streams.py | 2 +- .../connectors/source-harvest/source_harvest/streams.py | 2 +- .../connectors/source-hubspot/source_hubspot/streams.py | 2 +- .../connectors/source-intercom/source_intercom/source.py | 2 +- .../connectors/source-iterable/source_iterable/streams.py | 2 +- .../connectors/source-klaviyo/source_klaviyo/streams.py | 2 +- .../source-linkedin-ads/source_linkedin_ads/source.py | 2 +- .../connectors/source-mailchimp/source_mailchimp/streams.py | 2 +- .../connectors/source-marketo/source_marketo/source.py | 2 +- .../connectors/source-mixpanel/source_mixpanel/streams/base.py | 2 +- .../connectors/source-notion/source_notion/streams.py | 2 +- .../source_paypal_transaction/source.py | 2 +- .../connectors/source-pinterest/source_pinterest/source.py | 2 +- .../connectors/source-recharge/source_recharge/api.py | 2 +- .../connectors/source-salesforce/source_salesforce/streams.py | 2 +- .../connectors/source-sentry/source_sentry/streams.py | 2 +- .../connectors/source-slack/source_slack/source.py | 2 +- .../source_snapchat_marketing/source.py | 2 +- .../connectors/source-stripe/source_stripe/streams.py | 2 +- .../source-surveymonkey/source_surveymonkey/streams.py | 3 +-- .../source-tiktok-marketing/source_tiktok_marketing/streams.py | 2 +- .../connectors/source-twilio/source_twilio/streams.py | 3 +-- .../source-zendesk-chat/source_zendesk_chat/streams.py | 3 +-- .../source-zendesk-support/source_zendesk_support/streams.py | 3 +-- .../source-zendesk-talk/source_zendesk_talk/streams.py | 3 +-- 31 files changed, 31 insertions(+), 36 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 1a17e3362df0a..42e04b95dc849 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -5,9 +5,9 @@ from abc import ABC, abstractmethod from http import HTTPStatus from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy import requests +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import expand_refs diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py index 4a07ad0f07a05..d404c8daa4cad 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py @@ -15,8 +15,8 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream from .errors import HTTP_ERROR_CODES, error_msg_from_status diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index fca5b0297955a..34e2aa01c25fc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -12,11 +12,11 @@ import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.api import FacebookAdsApiBatch, FacebookRequest, FacebookResponse -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .common import deep_merge diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 34b0d896668ce..61367124566ab 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -11,10 +11,10 @@ import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.auth import AuthBase from source_freshdesk.utils import CallCredit diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 108f836ac8722..2398d8784e046 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -10,9 +10,9 @@ import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.exceptions import HTTPError from .graphql import CursorStorage, QueryReactions, get_query_pull_requests, get_query_reviews diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index 583bc14d401e2..56649bb11d931 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -17,9 +17,9 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .custom_reports_validator import CustomReportsValidator diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 82fc9fab4fa2b..8f5b54bb5d6a3 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -9,9 +9,9 @@ import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy BASE_URL = "https://www.googleapis.com/webmasters/v3/" ROW_LIMIT = 25000 diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index 6210db91dc4d6..f8872570b7275 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -9,8 +9,8 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream class HarvestStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 8b58eaf542c3d..0306dfb932be8 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -15,9 +15,9 @@ import requests from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests import codes from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 299890b1e764c..9ee1c8bd344b1 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -12,9 +12,9 @@ from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from requests.auth import AuthBase from .utils import EagerlyCachedStreamState as stream_state_cache diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index a57db43cc3266..bd0b530b01417 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -12,10 +12,10 @@ import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from pendulum.datetime import DateTime from requests import codes from requests.exceptions import ChunkedEncodingError diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 06960f2ba7c3e..e8ca965d49a3f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -8,9 +8,9 @@ import pendulum import requests +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class KlaviyoStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 77955917c3d2f..022a81d78cb54 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -14,10 +14,10 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .analytics import make_analytics_slices, merge_chunks, update_analytics_params from .utils import get_parent_stream_values, transform_data diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 63c569b0eea69..517e6c17f4c5b 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -6,10 +6,10 @@ import math from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 621f3f66f7be0..4c98eb43b3767 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -14,9 +14,9 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from .utils import STRING_TYPES, clean_string, format_value, to_datetime_str diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 2e839742f79d8..7f0e33ca3acf9 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -9,9 +9,9 @@ import pendulum import requests +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from pendulum import Date diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 6dc6e29949d5e..d2dc022612dd3 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -8,8 +8,8 @@ import pydantic import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from .utils import transform_properties diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index cfb35a8d61ec3..0f6ab5bbeb45f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -14,10 +14,10 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from dateutil.parser import isoparse from .utils import middle_date_slices diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index c74088fd2be32..244f022fce55f 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -13,8 +13,8 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 56f00b84d3cd3..bff8cf65247f4 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -8,9 +8,9 @@ import pendulum import requests +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy class RechargeStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 70c11eb9ab944..59b78b9a814aa 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -16,8 +16,8 @@ import requests # type: ignore[import] from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from numpy import nan from pendulum import DateTime # type: ignore[attr-defined] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 2a45bd9de59d8..d6a41a98ca5dc 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -9,8 +9,8 @@ import pendulum import requests from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream class SentryStream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index e58090ffe498f..7475dc7b9634f 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -8,10 +8,10 @@ import pendulum import requests +from airbte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource -from airbte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py index 26a92c7c328f0..790b7e2a2cc87 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py @@ -14,12 +14,12 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import IncrementalMixin, package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy # https://marketingapi.snapchat.com/docs/#core-metrics # https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 38499ef1ab727..c811dff429fd2 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -10,8 +10,8 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream STRIPE_ERROR_CODES: List = [ # stream requires additional permissions diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 372e65b47a422..3a5340b272aac 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -6,13 +6,12 @@ import urllib.parse from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy - import pendulum import requests import vcr from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream cache_file = tempfile.NamedTemporaryFile() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 8400798158049..f426710b7169f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -15,10 +15,10 @@ import pydantic import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer # TikTok Initial release date is September 2016 diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 0b1c7922a55ef..7dea15c222d18 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -12,10 +12,9 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy - from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from pendulum.datetime import DateTime from requests.auth import AuthBase diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index c4de25b49b8f7..92b4e9806a716 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -9,9 +9,8 @@ import pendulum import requests -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy - +from airbyte_cdk.sources.streams.http import HttpStream class Stream(HttpStream, ABC): diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 70cdd2654a9ce..d2139bed9593b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -21,6 +21,7 @@ import pytz import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException @@ -28,8 +29,6 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests.auth import AuthBase from requests_futures.sessions import PICKLE_ERROR, FuturesSession -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy - DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY: str = "_last_end_time" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py index c458b2a133d0d..793832d77c5fe 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py @@ -10,9 +10,8 @@ import pendulum as pendulum import requests -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy - +from airbyte_cdk.sources.streams.http import HttpStream class ZendeskTalkStream(HttpStream, ABC): From 4400f70559f6765837630e2cc441fb6fd534a9ad Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 10:20:05 -0600 Subject: [PATCH 36/40] Update dockerfiles from merge conflict --- airbyte-integrations/connectors/source-gitlab/Dockerfile | 2 +- airbyte-integrations/connectors/source-sentry/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index f3d921dc247b3..16e8b76a76f00 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 5d05b54b34075..d1e3bca98ce15 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-sentry From b755e7889b51d3feba7ec5f5d8004bcba4594979 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 10:29:45 -0600 Subject: [PATCH 37/40] Revert "Restore HttpAvailabilityStrategy as default (revert https://github.com/airbytehq/airbyte/pull/21488)" This reverts commit 8570f591870455158ae87f54f63e7fead4867f18. --- .../airbyte_cdk/sources/streams/http/http.py | 6 ------ .../python/docs/concepts/http-streams.md | 17 ++++++----------- .../declarative/checks/test_check_stream.py | 6 ++++++ .../streams/http/test_availability_strategy.py | 6 ++++++ .../python/unit_tests/sources/test_source.py | 11 +++++++++++ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 35cef60807fe9..084eb052894ed 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -13,9 +13,7 @@ import requests import requests_cache from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream, StreamData -from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests.auth import AuthBase from requests_cache.session import CachedSession @@ -115,10 +113,6 @@ def retry_factor(self) -> float: def authenticator(self) -> HttpAuthenticator: return self._authenticator - @property - def availability_strategy(self) -> Optional[AvailabilityStrategy]: - return HttpAvailabilityStrategy() - @abstractmethod def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ diff --git a/airbyte-cdk/python/docs/concepts/http-streams.md b/airbyte-cdk/python/docs/concepts/http-streams.md index 0008ea281153b..aa80d05fc266d 100644 --- a/airbyte-cdk/python/docs/concepts/http-streams.md +++ b/airbyte-cdk/python/docs/concepts/http-streams.md @@ -87,20 +87,15 @@ be returned as a keyword argument. The CDK defines an `AvailabilityStrategy` for a stream, which is used to perform the `check_availability` method. This method checks whether the stream is available before performing `read_records`. -For HTTP streams, a default `HttpAvailabilityStrategy` is defined, which attempts to read the first record of the stream, and excepts +For HTTP streams, a `HttpAvailabilityStrategy` is defined, which attempts to read the first record of the stream, and excepts a dictionary of known error codes and associated reasons, `reasons_for_unavailable_status_codes`. By default, this list contains only `requests.status_codes.FORBIDDEN` (403), with an associated error message that tells the user that they are likely missing permissions associated with that stream. -### Customizing stream availability - -You can subclass `HttpAvailabilityStrategy` to override the `reasons_for_unavailable_status_codes` to except more HTTP error codes and inform the user how to resolve errors specific to your connector or stream. - -### Disabling stream availability check - -You can disable the `HttpAvailabilityStrategy` in your `HttpStream` by adding the following property to your stream class: +You can use this `HttpAvailabilityStrategy` in your `HttpStream` by adding the following property to your stream class: ```python - @property def availability_strategy(self) -> Optional[AvailabilityStrategy]: - return None -``` \ No newline at end of file + return HttpAvailabilityStrategy() +``` + +You can also subclass `HttpAvailabilityStrategy` to override the list of known errors to except more error codes and inform the user how to resolve errors specific to your connector or stream. diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 2c5670a0896ee..52a2e4f03500b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -9,6 +9,7 @@ import pytest import requests from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy @@ -119,6 +120,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield stub_resp pass + # TODO (Ella): Remove explicit definition when turning on default + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return HttpAvailabilityStrategy() + http_stream = MockHttpStream() assert isinstance(http_stream, HttpStream) assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 4afed0fd11886..6bd1cf005fa26 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -9,6 +9,7 @@ import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.http import HttpStream from requests import HTTPError @@ -39,6 +40,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def retry_factor(self) -> float: return 0.01 + # TODO (Ella): Remove explicit definition when turning on default + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return HttpAvailabilityStrategy() + @pytest.mark.parametrize( ("status_code", "json_contents", "expected_is_available", "expected_messages"), diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index 08eb38cfa2aed..b8a8291058e82 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -23,6 +23,7 @@ Type, ) from airbyte_cdk.sources import AbstractSource, Source +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.http import HttpStream @@ -495,6 +496,11 @@ def __init__(self, *args, **kvargs): HttpStream.__init__(self, *args, kvargs) self.read_records = mocker.MagicMock() + # TODO (Ella): Remove explicit definition when turning on default + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return HttpAvailabilityStrategy() + class MockStream(mocker.MagicMock, Stream): page_size = None get_json_schema = mocker.MagicMock() @@ -548,6 +554,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp self.resp_counter += 1 yield stub_response + # TODO (Ella): Remove explicit definition when turning on default + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return HttpAvailabilityStrategy() + class MockStream(mocker.MagicMock, Stream): page_size = None get_json_schema = mocker.MagicMock() From 10da72ab3926b7f1bb1c37a344bb34a4fbc75662 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 10:53:08 -0600 Subject: [PATCH 38/40] Add missing changelog for linkedin-ads --- docs/integrations/sources/linkedin-ads.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 0752cb11fca7e..7399e8e983a11 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -182,7 +182,8 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +|:--------| :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 0.1.13 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.12 | 2022-10-18 | [18111](https://github.com/airbytehq/airbyte/pull/18111) | for adDirectSponsoredContents stream skip accounts which are part of organization | | 0.1.11 | 2022-10-07 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | Retry 429/5xx errors when refreshing access token | | 0.1.10 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | From 03424dda98d6f1c4d9ef29170657f38f9229f533 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 13:11:08 -0600 Subject: [PATCH 39/40] Fix typo in source slack import --- .../connectors/source-slack/source_slack/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 7475dc7b9634f..22c7f8f2380ba 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -8,11 +8,11 @@ import pendulum import requests -from airbte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from pendulum import DateTime, Period From 57a56a0ce6dd89668b4778b3123d0f2bf8ae173c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Jan 2023 13:16:01 -0600 Subject: [PATCH 40/40] Fix 2 changelog entries with incorrectly bumped versions --- docs/integrations/sources/harvest.md | 2 +- docs/integrations/sources/salesforce.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index 089c1bebcbe9f..95c9222200c0a 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -79,7 +79,7 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------| -| 0.1.25 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` || | | | | +| 0.1.15 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.14 | 2023-01-09 | [21151](https://github.com/airbytehq/airbyte/pull/21151) | Skip 403 FORBIDDEN for all stream | | 0.1.13 | 2022-12-22 | [20810](https://github.com/airbytehq/airbyte/pull/20810) | Skip 403 FORBIDDEN for `EstimateItemCategories` stream | | 0.1.12 | 2022-12-16 | [20572](https://github.com/airbytehq/airbyte/pull/20572) | Introduce replication end date | diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index e66551926a062..93ca47bf16a18 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -129,7 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| -| 0.1.30 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.30 | TODO | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.29 | 2023-01-05 | [20886](https://github.com/airbytehq/airbyte/pull/20886) | Remove `ActivityMetric` stream | | 1.0.28 | 2022-12-29 | [20927](https://github.com/airbytehq/airbyte/pull/20927) | Fix tests; add expected records | | 1.0.27 | 2022-11-29 | [19869](https://github.com/airbytehq/airbyte/pull/19869) | Remove `AccountHistory` from unsupported BULK streams |