diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 7a1e8e2d82abe..7dc8eb5652368 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.2.0 +LABEL io.airbyte.version=3.3.0 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index d0be492d33693..5f27efd459338 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -61,5 +61,7 @@ acceptance_tests: spec: tests: - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: "3.2.0" connector_image: airbyte/source-amazon-ads:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 715c567b4ce50..585149544f872 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -13,6 +13,7 @@ "client_id": { "title": "Client ID", "description": "The client ID of your Amazon Ads developer application. See the docs for more information.", + "airbyte_secret": true, "order": 1, "type": "string" }, @@ -43,7 +44,9 @@ "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", "examples": ["2022-10-10", "2022-10-22"], "order": 5, - "type": "string" + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "format": "date" }, "profiles": { "title": "Profile IDs", diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index fa309cc66e047..1c292d29e4cad 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -7,7 +7,9 @@ from airbyte_cdk.entrypoint import launch from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.config_migrations import MigrateStartDate if __name__ == "__main__": source = SourceAmazonAds() + MigrateStartDate.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index 517408077b889..458d14f52f01b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 3.2.0 + dockerImageTag: 3.3.0 dockerRepository: airbyte/source-amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py new file mode 100644 index 0000000000000..dda920775164d --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateStartDate: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Delete start_date field if it set to None or an empty string (""). + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + # Key used to identify start date in the configuration. + key: str = "start_date" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + Determines if a configuration requires migration. + + Args: + - config (Mapping[str, Any]): The configuration data to check. + + Returns: + - True: If the configuration requires migration. + - False: Otherwise. + """ + return not config.get(cls.key, "skip_if_start_date_in_config") + + @classmethod + def delete_from_config(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + """ + Removes the specified key from the configuration. + + Args: + - config (Mapping[str, Any]): The configuration from which the key should be removed. + - source (Source, optional): The data source. Defaults to None. + + Returns: + - Mapping[str, Any]: The configuration after removing the key. + """ + config.pop(cls.key, None) # Safely remove the key if it exists. + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Modifies the configuration and then saves it back to the source. + + Args: + - config_path (str): The path where the configuration is stored. + - source (Source): The data source. + - config (Mapping[str, Any]): The current configuration. + + Returns: + - Mapping[str, Any]: The updated configuration. + """ + migrated_config = cls.delete_from_config(config, source) + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + """ + Emits the control messages related to configuration migration. + + Args: + - migrated_config (Mapping[str, Any]): The migrated configuration. + """ + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the configuration migration process. + + It first checks if the `--config` argument is provided, and if so, + determines whether migration is needed, and then performs the migration + if required. + + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + if cls.should_migrate(config): + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py index dba006e1b1562..b7c375fefaf7f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py @@ -52,3 +52,8 @@ class BrandsAdGroup(CatalogModel): adGroupId: Decimal name: str bid: int + keywordId: Decimal + keywordText: str + nativeLanguageKeyword: str + matchType: str + state: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py index d60f1042592cd..b5ca604e06b69 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py @@ -32,6 +32,7 @@ class ProductCampaign(CatalogModel): endDate: str = None premiumBidAdjustment: bool bidding: Bidding + networks: str class ProductAdGroups(CatalogModel): diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 0df1bdd6d002c..09b4d52205f32 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -17,6 +17,7 @@ connectionSpecification: for more information. order: 1 type: string + airbyte_secret: true client_secret: title: Client Secret description: @@ -51,6 +52,8 @@ connectionSpecification: description: The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + format: date examples: - "2022-10-10" - "2022-10-22" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index 2b69cd22e4e4c..eb6cbb4dd93fd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config +from source_amazon_ads.config_migrations import MigrateStartDate def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: @@ -41,6 +42,7 @@ def read_full_refresh(stream_instance: Stream): def command_check(source: Source, config): logger = mock.MagicMock() connector_config, _ = split_config(config) + connector_config = MigrateStartDate.modify_and_save("unit_tests/config.json", source, connector_config) if source.check_config_against_spec: source_spec: ConnectorSpecification = source.spec(logger) check_config_against_spec_or_exit(connector_config, source_spec) diff --git a/docs/integrations/sources/amazon-ads.inapp.md b/docs/integrations/sources/amazon-ads.inapp.md deleted file mode 100644 index d501c7be46423..0000000000000 --- a/docs/integrations/sources/amazon-ads.inapp.md +++ /dev/null @@ -1,20 +0,0 @@ -## Prerequisites - -- An [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com) - -## Setup Guide - -1. Click `Authenticate your Amazon Ads account`. Log in and authorize access to the Amazon account. -2. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [Amazon Ads documentation](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -3. (Optional) **Start Date** can be used to generate reports starting from the specified start date in the format YYYY-MM-DD. The date should not be more than 60 days in the past. If not specified, today's date is used. The date is treated in the timezone of the processed profile. -4. (Optional) **Profile ID(s)** you want to fetch data for. A profile is an advertiser's account in a specific marketplace. See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. If not specified, data from all Profiles will be synced. -5. (Optional) **State Filter** Filter for Display, Product, and Brand Campaign streams with a state of enabled, paused, or archived. If not specified, all streams regardless of state will be synced. -6. (Optional) **Look Back Window** The amount of days to go back in time to get the updated data from Amazon Ads. After the first sync, data from this date will be synced. -7. (Optional) **Report Record Types** Optional configuration which accepts an array of string of record types. Leave blank for default behaviour to pull all report types. Use this config option only if you want to pull specific report type(s). See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/reporting/v2/report-types) for more details -9. Click `Set up source`. - -### Report Timezones - -All the reports are generated relative to the target profile' timezone. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Amazon Ads](https://docs.airbyte.com/integrations/sources/amazon-ads). diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 25dc221cdb6b0..680307eac4992 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -109,6 +109,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 3.3.0 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords` | | 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | | 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | | 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups |