Skip to content

Commit

Permalink
🎉 Source Amazon Ads: Skip API error "Report date is too far in the pa…
Browse files Browse the repository at this point in the history 
…st." (#15921)

* _check_report_date_error added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* amazon-ads.md updated

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* improve docs

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* improve logging, improve type hints

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* remove ReportInitFatal

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* amazon-ads.md updated

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* auto-bump connector version [ci skip]

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
  • Loading branch information
@grubberr @octavia-squidington-iii
grubberr and octavia-squidington-iii committed Aug 25, 2022
1 parent 2c89185 commit f952ace
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 13 deletions.
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
- name: Amazon Ads
sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
dockerRepository: airbyte/source-amazon-ads
dockerImageTag: 0.1.16
dockerImageTag: 0.1.17
documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
icon: amazonads.svg
sourceType: api
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_specs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-amazon-ads:0.1.16"
- dockerImage: "airbyte/source-amazon-ads:0.1.17"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
connectionSpecification:
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-amazon-ads/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.16
LABEL io.airbyte.version=0.1.17
LABEL io.airbyte.name=airbyte/source-amazon-ads
35 changes: 25 additions & 10 deletions ...s/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import backoff
import pendulum
import requests
from airbyte_cdk.logger import AirbyteLogger
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator
from pendulum import Date
Expand All @@ -24,8 +23,6 @@
from source_amazon_ads.streams.common import BasicAmazonAdsStream
from source_amazon_ads.utils import iterate_one_by_one

logger = AirbyteLogger()


class RecordType(str, Enum):
CAMPAIGNS = "campaigns"
Expand Down Expand Up @@ -176,14 +173,14 @@ def wrapped(self, *args, **kwargs):
@backoff_max_tries
def _init_and_try_read_records(self, profile: Profile, report_date):
report_infos = self._init_reports(profile, report_date)
logger.info(f"Waiting for {len(report_infos)} report(s) to be generated")
self.logger.info(f"Waiting for {len(report_infos)} report(s) to be generated")
self._try_read_records(report_infos)
return report_infos

@backoff_max_time
def _try_read_records(self, report_infos):
incomplete_report_infos = self._incomplete_report_infos(report_infos)
logger.info(f"Checking report status, {len(incomplete_report_infos)} report(s) remaining")
self.logger.info(f"Checking report status, {len(incomplete_report_infos)} report(s) remaining")
for report_info in incomplete_report_infos:
report_status, download_url = self._check_status(report_info)
report_info.status = report_status
Expand Down Expand Up @@ -358,16 +355,18 @@ def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]:
# subtypes have mutualy excluded parameters so we requesting
# different metric list for each record.
record_type = record_type.split("_")[0]
logger.info(f"Initiating report generation for {profile.profileId} profile with {record_type} type for {report_date} date")
self.logger.info(f"Initiating report generation for {profile.profileId} profile with {record_type} type for {report_date} date")
response = self._send_http_request(
urljoin(self._url, self.report_init_endpoint(record_type)),
profile.profileId,
report_init_body,
)
if response.status_code != HTTPStatus.ACCEPTED:
raise ReportInitFailure(
f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}"
)
error_msg = f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}"
if self._check_report_date_error(response):
self.logger.warning(error_msg)
break
raise ReportInitFailure(error_msg)

response = ReportInitResponse.parse_raw(response.text)
report_infos.append(
Expand All @@ -379,7 +378,7 @@ def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]:
metric_objects=[],
)
)
logger.info("Initiated successfully")
self.logger.info("Initiated successfully")

return report_infos

Expand All @@ -401,3 +400,19 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]:
if isinstance(exception, ReportGenerationInProgress):
return f'Report(s) generation time took more than {self.report_wait_timeout} minutes, please increase the "report_wait_timeout" parameter in configuration.'
return super().get_error_display_message(exception)

def _check_report_date_error(self, response) -> bool:
"""
Check if the connector received an error: 'Report date is too far in the past. Reports are only available for 60 days.'
In theory, it does not have to get such an error because the connector correctly calculates the start date,
but from practice, we can still catch such errors from time to time.
"""

if response.status_code == 406:
try:
response_json = response.json()
except ValueError:
return False
return response_json.get("details", "").startswith("Report date is too far in the past.")
return False
1 change: 1 addition & 0 deletions docs/integrations/sources/amazon-ads.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ Information about expected report generation waiting time you may find [here](ht

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------|
| 0.1.17 | 2022-08-24 | [15921](https://github.com/airbytehq/airbyte/pull/15921) | Skip API error "Report date is too far in the past." |
| 0.1.16 | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822) | Set default value for 'region' if needed |
| 0.1.15 | 2022-08-20 | [15816](https://github.com/airbytehq/airbyte/pull/15816) | Update STATE of incremental sync if no records |
| 0.1.14 | 2022-08-15 | [15637](https://github.com/airbytehq/airbyte/pull/15637) | Generate slices by lazy evaluation |
Expand Down

0 comments on commit f952ace

Please sign in to comment.