From 99079f92511bbaa452e3d81686d8e8e2115f7e8b Mon Sep 17 00:00:00 2001 From: askarpets Date: Wed, 21 Feb 2024 14:32:31 +0200 Subject: [PATCH 1/5] Source Amazon Ads: migrate source to YamlDeclarativeSource with custom check_connection --- .../declarative_source_adapter.py | 37 ++++ .../source_amazon_ads/manifest.yaml | 169 ++++++++++++++++++ .../source_amazon_ads/run.py | 3 +- .../source_amazon_ads/spec.yaml | 165 ----------------- 4 files changed, 208 insertions(+), 166 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py new file mode 100644 index 0000000000000..ede81f77e8cb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from logging import Logger +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import AirbyteConnectionStatus +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog + + +class DeclarativeSourceAdapter(YamlDeclarativeSource): + def __init__(self, source: AbstractSource) -> None: + super().__init__(path_to_yaml="manifest.yaml") + self._source = source + + def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + return self._source.check(logger, config) + + def discover(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteCatalog: + return self._source.discover(logger, config) + + def read( + self, + logger: Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, + ) -> Iterator[AirbyteMessage]: + return self._source.read(logger, config, catalog, state) + + # def _validate_source(self) -> None: + # # TODO ??? + # return diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml new file mode 100644 index 0000000000000..b7e8c86d7e3d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml @@ -0,0 +1,169 @@ +version: 0.60.1 +type: source_amazon_ads.SourceAmazonAds +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/amazon-ads + connection_specification: + title: Amazon Ads Spec + type: object + properties: + auth_type: + title: Auth Type + const: oauth2.0 + order: 0 + type: string + client_id: + title: Client ID + description: + The client ID of your Amazon Ads developer application. See the + docs + for more information. + order: 1 + type: string + airbyte_secret: true + client_secret: + title: Client Secret + description: + The client secret of your Amazon Ads developer application. See + the docs + for more information. + airbyte_secret: true + order: 2 + type: string + refresh_token: + title: Refresh Token + description: + Amazon Ads refresh token. See the docs + for more information on how to obtain this token. + airbyte_secret: true + order: 3 + type: string + region: + title: Region + description: + Region to pull data from (EU/NA/FE). See docs + for more details. + enum: + - NA + - EU + - FE + type: string + default: NA + order: 4 + start_date: + title: Start Date + description: + The Start date for collecting reports, should not be more than + 60 days in the past. In YYYY-MM-DD format + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + format: date + examples: + - "2022-10-10" + - "2022-10-22" + order: 5 + type: string + profiles: + title: Profile IDs + description: 'Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.' + order: 6 + type: array + items: + type: integer + marketplace_ids: + title: Marketplace IDs + description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID." + order: 7 + type: array + items: + type: string + state_filter: + title: State Filter + description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. + items: + type: string + enum: + - enabled + - paused + - archived + type: array + uniqueItems: true + order: 8 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 9 + report_record_types: + title: Report Record Types + description: + Optional configuration which accepts an array of string of record types. + Leave blank for default behaviour to pull all report types. + Use this config option only if you want to pull specific report type(s). + See docs + for more details + items: + type: string + enum: + - adGroups + - asins + - asins_keywords + - asins_targets + - campaigns + - keywords + - productAds + - targets + type: array + uniqueItems: true + order: 10 + required: + - client_id + - client_secret + - refresh_token + additionalProperties: true + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - auth_type + predicate_value: oauth2.0 + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + region: + type: string + path_in_connector_config: + - region + complete_oauth_output_specification: + type: object + additionalProperties: true + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret +streams: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py index a8012240de66d..0436d379599e3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -8,9 +8,10 @@ from airbyte_cdk.entrypoint import launch from source_amazon_ads import SourceAmazonAds from source_amazon_ads.config_migrations import MigrateStartDate +from source_amazon_ads.declarative_source_adapter import DeclarativeSourceAdapter def run(): - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) MigrateStartDate.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml deleted file mode 100644 index 0e703cb4ca3af..0000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ /dev/null @@ -1,165 +0,0 @@ ---- -documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads -connectionSpecification: - title: Amazon Ads Spec - type: object - properties: - auth_type: - title: Auth Type - const: oauth2.0 - order: 0 - type: string - client_id: - title: Client ID - description: - The client ID of your Amazon Ads developer application. See the - docs - for more information. - order: 1 - type: string - airbyte_secret: true - client_secret: - title: Client Secret - description: - The client secret of your Amazon Ads developer application. See - the docs - for more information. - airbyte_secret: true - order: 2 - type: string - refresh_token: - title: Refresh Token - description: - Amazon Ads refresh token. See the docs - for more information on how to obtain this token. - airbyte_secret: true - order: 3 - type: string - region: - title: Region - description: - Region to pull data from (EU/NA/FE). See docs - for more details. - enum: - - NA - - EU - - FE - type: string - default: NA - order: 4 - start_date: - title: Start Date - description: - The Start date for collecting reports, should not be more than - 60 days in the past. In YYYY-MM-DD format - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - format: date - examples: - - "2022-10-10" - - "2022-10-22" - order: 5 - type: string - profiles: - title: Profile IDs - description: 'Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.' - order: 6 - type: array - items: - type: integer - marketplace_ids: - title: Marketplace IDs - description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID." - order: 7 - type: array - items: - type: string - state_filter: - title: State Filter - description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. - items: - type: string - enum: - - enabled - - paused - - archived - type: array - uniqueItems: true - order: 8 - look_back_window: - title: "Look Back Window" - description: "The amount of days to go back in time to get the updated data from Amazon Ads" - examples: - - 3 - - 10 - type: "integer" - default: 3 - order: 9 - report_record_types: - title: Report Record Types - description: - Optional configuration which accepts an array of string of record types. - Leave blank for default behaviour to pull all report types. - Use this config option only if you want to pull specific report type(s). - See docs - for more details - items: - type: string - enum: - - adGroups - - asins - - asins_keywords - - asins_targets - - campaigns - - keywords - - productAds - - targets - type: array - uniqueItems: true - order: 10 - required: - - client_id - - client_secret - - refresh_token - additionalProperties: true -advanced_auth: - auth_flow_type: oauth2.0 - predicate_key: - - auth_type - predicate_value: oauth2.0 - oauth_config_specification: - oauth_user_input_from_connector_config_specification: - type: object - additionalProperties: false - properties: - region: - type: string - path_in_connector_config: - - region - complete_oauth_output_specification: - type: object - additionalProperties: true - properties: - refresh_token: - type: string - path_in_connector_config: - - refresh_token - complete_oauth_server_input_specification: - type: object - additionalProperties: true - properties: - client_id: - type: string - client_secret: - type: string - complete_oauth_server_output_specification: - type: object - additionalProperties: true - properties: - client_id: - type: string - path_in_connector_config: - - client_id - client_secret: - type: string - path_in_connector_config: - - client_secret From 5e7e9c5d273de0bf89941df5a8faa191f868af07 Mon Sep 17 00:00:00 2001 From: askarpets Date: Thu, 22 Feb 2024 13:29:43 +0200 Subject: [PATCH 2/5] Update DeclarativeSourceAdapter --- .../source-amazon-ads/metadata.yaml | 2 +- .../source-amazon-ads/pyproject.toml | 2 +- .../declarative_source_adapter.py | 36 ++++++++++--------- .../source_amazon_ads/manifest.yaml | 1 - .../source_amazon_ads/source.py | 13 +++---- docs/integrations/sources/amazon-ads.md | 7 ++-- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index c12b2c7341c39..dc62a61bca6bf 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 4.0.3 + dockerImageTag: 4.0.4 dockerRepository: airbyte/source-amazon-ads documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads githubIssueLabel: source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml b/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml index 0d00d8b9f3144..e281ad72d5ab7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml +++ b/airbyte-integrations/connectors/source-amazon-ads/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.0.3" +version = "4.0.4" name = "source-amazon-ads" description = "Source implementation for Amazon Ads." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py index ede81f77e8cb1..6bd8e62b56197 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py @@ -4,34 +4,36 @@ from logging import Logger -from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Union +from typing import Any, List, Mapping from airbyte_cdk.models import AirbyteConnectionStatus from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.sources.streams import Stream +from airbyte_protocol.models import ConnectorSpecification class DeclarativeSourceAdapter(YamlDeclarativeSource): def __init__(self, source: AbstractSource) -> None: super().__init__(path_to_yaml="manifest.yaml") self._source = source + self._set_adapted_methods() + + def spec(self, logger: Logger) -> ConnectorSpecification: + return self._source.spec(logger) def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: return self._source.check(logger, config) - def discover(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteCatalog: - return self._source.discover(logger, config) - - def read( - self, - logger: Logger, - config: Mapping[str, Any], - catalog: ConfiguredAirbyteCatalog, - state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, - ) -> Iterator[AirbyteMessage]: - return self._source.read(logger, config, catalog, state) - - # def _validate_source(self) -> None: - # # TODO ??? - # return + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return self._source.streams(config) + + def _validate_source(self) -> None: + """Skipping manifest validation as it can be incomplete when use adapter""" + return + + def _set_adapted_methods(self) -> None: + adapted_methods = ("spec", "check", "streams") + for method in adapted_methods: + if method in self.resolved_manifest: + self._source.__setattr__(method, getattr(super(), method)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml index b7e8c86d7e3d0..5dd0d541cb346 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/manifest.yaml @@ -166,4 +166,3 @@ spec: type: string path_in_connector_config: - client_secret -streams: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 9d1852c33330a..1eac1615faeee 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -50,7 +50,7 @@ class SourceAmazonAds(AbstractSource): - def _validate_and_transform(self, config: Mapping[str, Any]): + def _validate_and_transform(self, config: Mapping[str, Any]) -> Mapping[str, Any]: start_date = config.get("start_date") if start_date: config["start_date"] = pendulum.from_format(start_date, CONFIG_DATE_FORMAT).date() @@ -69,7 +69,8 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> """ :param config: the user-input config object conforming to the connector's spec.json :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, + (False, error) otherwise. """ try: config = self._validate_and_transform(config) @@ -78,7 +79,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> # Check connection by sending list of profiles request. Its most simple # request, not require additional parameters and usually has few data # in response body. - # It doesnt support pagination so there is no sense of reading single + # It doesn't support pagination so there is no sense of reading single # record, it would fetch all the data anyway. profiles_list = Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles() filtered_profiles = self._choose_profiles(config, profiles_list) @@ -89,15 +90,15 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. - :return list of streams for current source + :return: list of streams for current source """ config = self._validate_and_transform(config) auth = self._make_authenticator(config) stream_args = {"config": config, "authenticator": auth} # All data for individual Amazon Ads stream divided into sets of data for # each profile. Every API request except profiles has required - # paramater passed over "Amazon-Advertising-API-Scope" http header and - # should contain profile id. So every stream is dependant on Profiles + # parameter passed over "Amazon-Advertising-API-Scope" http header and + # should contain profile id. So every stream is dependent on Profiles # stream and should have information about all profiles. profiles_stream = Profiles(**stream_args) profiles_list = profiles_stream.get_all_profiles() diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 532712344b5e2..275f2d38a5351 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -110,7 +110,8 @@ Information about expected report generation waiting time can be found [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 4.0.3 | 2024-02-12 | [35180](https://github.com/airbytehq/airbyte/pull/35180) | Manage dependencies with Poetry. | +| 4.0.4 | 2024-02-23 | [35481](https://github.com/airbytehq/airbyte/pull/35481) | Migrate source to `YamlDeclarativeSource` with custom `check_connection` | +| 4.0.3 | 2024-02-12 | [35180](https://github.com/airbytehq/airbyte/pull/35180) | Manage dependencies with Poetry | | 4.0.2 | 2024-02-08 | [35013](https://github.com/airbytehq/airbyte/pull/35013) | Add missing field to `sponsored_display_budget_rules` stream | | 4.0.1 | 2023-12-28 | [33833](https://github.com/airbytehq/airbyte/pull/33833) | Updated oauth spec to put region, so we can choose oauth consent url based on it | | 4.0.0 | 2023-12-28 | [33817](https://github.com/airbytehq/airbyte/pull/33817) | Fix schema for streams: `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` | @@ -120,7 +121,7 @@ Information about expected report generation waiting time can be found [here](ht | 3.3.0 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords` | | 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | | 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | -| 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups | +| 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no associated with bid ad groups | | 3.1.0 | 2023-08-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add `T00030` tactic support for `sponsored_display_report_stream` | | 3.0.0 | 2023-07-24 | [27868](https://github.com/airbytehq/airbyte/pull/27868) | Fix attribution report stream schemas | | 2.3.1 | 2023-07-11 | [28155](https://github.com/airbytehq/airbyte/pull/28155) | Bugfix: validation error when record values are missing | @@ -161,7 +162,7 @@ Information about expected report generation waiting time can be found [here](ht | 0.1.6 | 2022-04-20 | [11659](https://github.com/airbytehq/airbyte/pull/11659) | Add adId to products report | | 0.1.5 | 2022-04-08 | [11430](https://github.com/airbytehq/airbyte/pull/11430) | Add support OAuth2.0 | | 0.1.4 | 2022-02-21 | [10513](https://github.com/airbytehq/airbyte/pull/10513) | Increasing REPORT_WAIT_TIMEOUT for supporting report generation which takes longer time | -| 0.1.3 | 2021-12-28 | [8388](https://github.com/airbytehq/airbyte/pull/8388) | Add retry if recoverable error occured for reporting stream processing | +| 0.1.3 | 2021-12-28 | [8388](https://github.com/airbytehq/airbyte/pull/8388) | Add retry if recoverable error occurred for reporting stream processing | | 0.1.2 | 2021-10-01 | [6367](https://github.com/airbytehq/airbyte/pull/6461) | Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback | | 0.1.1 | 2021-09-22 | [6367](https://github.com/airbytehq/airbyte/pull/6367) | Add seller and vendor filters to profiles stream | | 0.1.0 | 2021-08-13 | [5023](https://github.com/airbytehq/airbyte/pull/5023) | Initial version | From 95ca569234d006e8fbdda908c95b1357e526b1f3 Mon Sep 17 00:00:00 2001 From: askarpets Date: Thu, 22 Feb 2024 14:47:09 +0200 Subject: [PATCH 3/5] Update tests --- .../unit_tests/integrations/utils.py | 3 +- .../unit_tests/test_source.py | 59 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py index 6d2828a3bf702..fafd5b37f7858 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py @@ -9,6 +9,7 @@ from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import SyncMode from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.declarative_source_adapter import DeclarativeSourceAdapter def read_stream( @@ -19,7 +20,7 @@ def read_stream( expecting_exception: bool = False ) -> EntrypointOutput: catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build() - return read(SourceAmazonAds(), config, catalog, state, expecting_exception) + return read(DeclarativeSourceAdapter(source=SourceAmazonAds()), config, catalog, state, expecting_exception) def get_log_messages_by_log_level(logs: List[AirbyteMessage], log_level: LogLevel) -> List[str]: diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index f3e8e9d93954e..4ecc750995547 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -7,6 +7,7 @@ from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type from jsonschema import Draft4Validator from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.declarative_source_adapter import DeclarativeSourceAdapter from source_amazon_ads.schemas import Profile from .utils import command_check, url_strip_query @@ -40,7 +41,7 @@ def ensure_additional_property_is_boolean(root): @responses.activate def test_discover(config): setup_responses() - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -50,7 +51,7 @@ def test_discover(config): def test_spec(): - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) spec = source.spec(None) assert isinstance(spec, ConnectorSpecification) @@ -58,7 +59,7 @@ def test_spec(): @responses.activate def test_check(config_gen): setup_responses() - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) assert len(responses.calls) == 2 @@ -89,36 +90,34 @@ def test_check(config_gen): @responses.activate def test_source_streams(config): setup_responses() - source = SourceAmazonAds() + source = DeclarativeSourceAdapter(source=SourceAmazonAds()) streams = source.streams(config) assert len(streams) == 29 actual_stream_names = {stream.name for stream in streams} - expected_stream_names = set( - [ - "profiles", - "portfolios", - "sponsored_display_campaigns", - "sponsored_product_campaigns", - "sponsored_product_ad_groups", - "sponsored_product_ad_group_suggested_keywords", - "sponsored_product_ad_group_bid_recommendations", - "sponsored_product_keywords", - "sponsored_product_negative_keywords", - "sponsored_product_campaign_negative_keywords", - "sponsored_product_ads", - "sponsored_product_targetings", - "sponsored_products_report_stream", - "sponsored_brands_campaigns", - "sponsored_brands_ad_groups", - "sponsored_brands_keywords", - "sponsored_brands_report_stream", - "attribution_report_performance_adgroup", - "attribution_report_performance_campaign", - "attribution_report_performance_creative", - "attribution_report_products", - "sponsored_display_budget_rules", - ] - ) + expected_stream_names = { + "profiles", + "portfolios", + "sponsored_display_campaigns", + "sponsored_product_campaigns", + "sponsored_product_ad_groups", + "sponsored_product_ad_group_suggested_keywords", + "sponsored_product_ad_group_bid_recommendations", + "sponsored_product_keywords", + "sponsored_product_negative_keywords", + "sponsored_product_campaign_negative_keywords", + "sponsored_product_ads", + "sponsored_product_targetings", + "sponsored_products_report_stream", + "sponsored_brands_campaigns", + "sponsored_brands_ad_groups", + "sponsored_brands_keywords", + "sponsored_brands_report_stream", + "attribution_report_performance_adgroup", + "attribution_report_performance_campaign", + "attribution_report_performance_creative", + "attribution_report_products", + "sponsored_display_budget_rules", + } assert not expected_stream_names - actual_stream_names From bd93b904c6c83d4017b112e92c16c3b703b09fcc Mon Sep 17 00:00:00 2001 From: askarpets Date: Fri, 1 Mar 2024 14:19:56 +0200 Subject: [PATCH 4/5] Add comment --- .../source_amazon_ads/declarative_source_adapter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py index 6bd8e62b56197..b81f6be509a10 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py @@ -33,6 +33,11 @@ def _validate_source(self) -> None: return def _set_adapted_methods(self) -> None: + """ + Since the adapter is intended to smoothly migrate the connector, + this method determines whether each of methods `spec`, `check`, and `streams` was declared in the manifest file + and if yes, makes the source use it, otherwise the method defined in the source will be used + """ adapted_methods = ("spec", "check", "streams") for method in adapted_methods: if method in self.resolved_manifest: From 3b06090660ac63f443b397c5c50652a16aad2adc Mon Sep 17 00:00:00 2001 From: askarpets Date: Tue, 5 Mar 2024 14:01:20 +0200 Subject: [PATCH 5/5] Add source name override --- .../source_amazon_ads/declarative_source_adapter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py index b81f6be509a10..6b46127b0b6bd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/declarative_source_adapter.py @@ -15,10 +15,14 @@ class DeclarativeSourceAdapter(YamlDeclarativeSource): def __init__(self, source: AbstractSource) -> None: - super().__init__(path_to_yaml="manifest.yaml") self._source = source + super().__init__(path_to_yaml="manifest.yaml") self._set_adapted_methods() + @property + def name(self) -> str: + return self._source.name + def spec(self, logger: Logger) -> ConnectorSpecification: return self._source.spec(logger)