From 59e588ac69c99af9a44fa319ddd132feee10369d Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Tue, 30 May 2023 21:56:47 +0200 Subject: [PATCH] Source Instagram: handle permission error (#26767) * Source Instagram: handle permission error * Source Instagram: update docs * Source Instagram: reduce max_retry -> 5; remove retry for 2108006 --- .../connectors/source-instagram/Dockerfile | 2 +- .../connectors/source-instagram/metadata.yaml | 2 +- .../source-instagram/source_instagram/api.py | 2 +- .../source_instagram/common.py | 2 +- .../source_instagram/streams.py | 3 ++ .../unit_tests/test_streams.py | 31 ++++++++++++++----- docs/integrations/sources/instagram.md | 5 +-- 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile index 6e8c5ce82b034..534ae8018452e 100644 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ b/airbyte-integrations/connectors/source-instagram/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.7 +LABEL io.airbyte.version=1.0.8 LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index d50b6c36fe6f6..69c082ce39043 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.7 + dockerImageTag: 1.0.8 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index fe3ae9c424e8e..1f73d04601414 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -17,7 +17,7 @@ from facebook_business.exceptions import FacebookRequestError from source_instagram.common import InstagramAPIException, retry_pattern -backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=7, factor=5) +backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) class MyFacebookAdsApi(FacebookAdsApi): diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py index 92b14439a1500..1899bbead0237 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py @@ -57,7 +57,7 @@ def should_retry_api_error(exc: FacebookRequestError): # The media was posted before the most recent time that the user's account # was converted to a business account from a personal account. if exc.api_error_type() == "OAuthException" and exc.api_error_code() == 100 and exc.api_error_subcode() == 2108006: - return True + return False return False diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index f6cc34fcd0640..0a90d9be93600 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -375,6 +375,9 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: # We receive all Media starting from the last one, and if on the next Media we get an Insight error, # then no reason to make inquiries for each Media further, since they were published even earlier. return None + elif error.api_error_code() == 100 and error.api_error_subcode() == 33: + self.logger.error(f"Check provided permissions for {account_id}: {error.api_error_message()}") + return None raise error diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index e7f5c01f760be..81bdff81e4d83 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -75,8 +75,9 @@ def test_media_insights_read(api, user_stories_data, user_media_insights_data, r def test_media_insights_read_error(api, requests_mock): test_id = "test_id" stream = MediaInsights(api=api) - media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}] + media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}] requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/media", json={"data": media_response}) + media_insights_response_test_id = { "name": "impressions", "period": "lifetime", @@ -86,7 +87,8 @@ def test_media_insights_read_error(api, requests_mock): "id": "test_id/insights/impressions/lifetime", } requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", json=media_insights_response_test_id) - error_response = { + + error_response_oauth = { "error": { "message": "Invalid parameter", "type": "OAuthException", @@ -96,11 +98,26 @@ def test_media_insights_read_error(api, requests_mock): "is_transient": False, "error_user_title": "Media posted before business account conversion", "error_user_msg": "The media was posted before the most recent time that the user's account was converted to a business account from a personal account.", - "fbtrace_id": "AJiAbw1WoyMzMUhBTxJqpxO" + "fbtrace_id": "fake_trace_id" + } + } + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_2/insights", json=error_response_oauth, status_code=400) + + error_response_wrong_permissions = { + "error": { + "message": "Invalid parameter", + "type": "OAuthException", + "code": 100, + "error_data": {}, + "error_subcode": 33, + "is_transient": False, + "error_user_msg": "Unsupported get request. Object with ID 'test_id_3' does not exist, cannot be loaded due to missing permissions, or does not support this operation.", + "fbtrace_id": "fake_trace_id" } } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_2/insights", json=error_response, status_code=400) - media_insights_response_test_id_3 = { + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_3/insights", json=error_response_wrong_permissions, status_code=400) + + media_insights_response_test_id_4 = { "name": "impressions", "period": "lifetime", "values": [{"value": 300}], @@ -108,7 +125,7 @@ def test_media_insights_read_error(api, requests_mock): "description": "Total number of times the media object has been seen", "id": "test_id_3/insights/impressions/lifetime", } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_3/insights", json=media_insights_response_test_id_3) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_4/insights", json=media_insights_response_test_id_4) records = read_full_refresh(stream) expected_records = [{"business_account_id": "test_id", @@ -116,7 +133,7 @@ def test_media_insights_read_error(api, requests_mock): "impressions": 264, "page_id": "act_unknown_account"}, {"business_account_id": "test_id", - "id": "test_id_3", + "id": "test_id_4", "impressions": 300, "page_id": "act_unknown_account"}] assert records == expected_records diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index d1ee653ead85e..9822b891bfdd8 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -71,7 +71,7 @@ Instagram limits the number of requests that can be made at a time, but the Inst AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/) system. This means that all sources must produce schemas and records within these types and all destinations must handle records that conform to this type system. | Integration Type | Airbyte Type | -| :--------------- | :----------- | +|:-----------------|:-------------| | `string` | `string` | | `number` | `number` | | `array` | `array` | @@ -82,8 +82,9 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for insights | | 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove authSpecification from connector specification in favour of advancedAuth | -| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Media posted before business account conversion | +| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | | 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | | 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | | 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream |