From d093e46bc3a6d2fded4fd2f223173fef667e032b Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 3 Oct 2023 02:21:25 +0100 Subject: [PATCH 01/11] fix sectionscompact stream, add sections stream --- .../connectors/source-asana/BOOTSTRAP.md | 2 +- .../integration_tests/configured_catalog.json | 9 +++++++++ .../source_asana/schemas/sections_compact.json | 14 ++++++++++++++ .../connectors/source-asana/source_asana/source.py | 3 ++- .../source-asana/source_asana/streams.py | 11 +++++++++-- .../source-asana/unit_tests/test_streams.py | 4 ++-- 6 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json diff --git a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md index 2fc7a562ca16a..9be462491a184 100644 --- a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md +++ b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md @@ -6,7 +6,7 @@ Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-de Some streams depend on: - workspaces (Teams, Users, CustomFields, Projects, Tags, Users streams); -- projects (Sections, Tasks streams); +- projects (SectionsCompact, Sections, Tasks streams); - tasks (Stories stream); - teams (TeamMemberships stream). diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json index 104d114b919af..3e9a2f61f9ecf 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json @@ -9,6 +9,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sections_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sections", diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json new file mode 100644 index 0000000000000..39d0b95b5bf84 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json @@ -0,0 +1,14 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 7c781fc1e0bf7..e9fb77e0783f2 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator -from .streams import CustomFields, Projects, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces +from .streams import CustomFields, Projects, SectionsCompact, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces class SourceAsana(AbstractSource): @@ -46,6 +46,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ CustomFields(**args), Projects(**args), + SectionsCompact(**args), Sections(**args), Stories(**args), Tags(**args), diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 081d2a140ed92..9fb11dfd5cc27 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -132,7 +132,7 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Mu class ProjectRelatedStream(AsanaStream, ABC): """ - Few streams (Sections and Tasks) depends on `project gid`: Sections as a part of url and Tasks as `projects` + Few streams (SectionsCompact and Tasks) depends on `project gid`: SectionsCompact as a part of url and Tasks as `projects` argument in request. """ @@ -153,11 +153,18 @@ def path(self, **kwargs) -> str: return "projects" -class Sections(ProjectRelatedStream): +class SectionsCompact(ProjectRelatedStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: project_gid = stream_slice["project_gid"] return f"projects/{project_gid}/sections" +class Section(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + section_gid = stream_slice["section_gid"] + return f"sections/{section_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=SectionsCompact, slice_field="section_gid") class Stories(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py index 377a2e3f5181e..5113d0fd37e01 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py @@ -7,12 +7,12 @@ import pytest import requests_mock as req_mock from airbyte_cdk.models import SyncMode -from source_asana.streams import AsanaStream, Sections, Stories, Tags, Tasks, TeamMemberships, Users +from source_asana.streams import AsanaStream, SectionsCompact, Sections, Stories, Tags, Tasks, TeamMemberships, Users @pytest.mark.parametrize( "stream", - [Tasks, Sections, Users, TeamMemberships, Tags, Stories], + [Tasks, SectionsCompact, Sections, Users, TeamMemberships, Tags, Stories], ) def test_task_stream(requests_mock, stream, mock_response): requests_mock.get(req_mock.ANY, json=mock_response) From 94b0132798942eceef767471f18e7e76eeef7d8a Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 9 Oct 2023 03:50:26 -0400 Subject: [PATCH 02/11] fix: bump versioning and number of streams in unit test --- airbyte-integrations/connectors/source-asana/Dockerfile | 2 +- airbyte-integrations/connectors/source-asana/metadata.yaml | 2 +- .../connectors/source-asana/unit_tests/test_source.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index 07ff709fff050..5fd80f2605522 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 00a96f584b23b..1c6e0f19fe188 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/source-asana documentationUrl: https://docs.airbyte.com/integrations/sources/asana githubIssueLabel: source-asana diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py index 7838cd23b8d56..d6353644c9e82 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py @@ -38,4 +38,4 @@ def test_check_connection_exception(config): def test_streams(config): streams = SourceAsana().streams(config) - assert len(streams) == 10 + assert len(streams) == 11 From 80b82a98e4717b2c48ea27af9f0dd33ec0da9f70 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 9 Oct 2023 03:52:50 -0400 Subject: [PATCH 03/11] format: run black on streams.py --- .../connectors/source-asana/source_asana/streams.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 9fb11dfd5cc27..eabf7161c542e 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -158,14 +158,16 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: project_gid = stream_slice["project_gid"] return f"projects/{project_gid}/sections" + class Section(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: section_gid = stream_slice["section_gid"] return f"sections/{section_gid}" - + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=SectionsCompact, slice_field="section_gid") + class Stories(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: task_gid = stream_slice["task_gid"] From 6b2d322bc3f02801f261abf9abfca6d044aac968 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 11:22:59 -0400 Subject: [PATCH 04/11] docs: add entry to changelog --- docs/integrations/sources/asana.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index dede4313e1a3e..0d9b94d7e6b63 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -68,6 +68,7 @@ The connector is restricted by normal Asana [requests limitation](https://develo | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.8 | 2023-10-16 | [31009](https://github.com/airbytehq/airbyte/pull/31009) | Add SectionsCompact stream | | 0.1.7 | 2023-05-29 | [26716](https://github.com/airbytehq/airbyte/pull/26716) | Remove authSpecification from spec.json, use advancedAuth instead | | 0.1.6 | 2023-05-26 | [26653](https://github.com/airbytehq/airbyte/pull/26653) | Fix order of authentication methods | | 0.1.5 | 2022-11-16 | [19561](https://github.com/airbytehq/airbyte/pull/19561) | Added errors handling, updated SAT with new format | From 3796cabdfd315486d25369d1159a200bcd022466 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 11:35:10 -0400 Subject: [PATCH 05/11] fix: correct declaration of sections stream --- .../connectors/source-asana/source_asana/streams.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index eabf7161c542e..90f37f7826d24 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -74,7 +74,8 @@ def get_opt_fields(self) -> MutableMapping[str, str]: if "object" in value["type"]: opt_fields.append(self._handle_object_type(prop, value)) elif "array" in value["type"]: - opt_fields.append(self._handle_array_type(prop, value.get("items", []))) + opt_fields.append(self._handle_array_type( + prop, value.get("items", []))) else: opt_fields.append(prop) @@ -91,7 +92,8 @@ def _handle_array_type(self, prop: str, value: MutableMapping[str, Any]) -> str: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() - yield from response_json.get("data", []) # Asana puts records in a container array "data" + # Asana puts records in a container array "data" + yield from response_json.get("data", []) def read_slices_from_records(self, stream_class: AsanaStreamType, slice_field: str) -> Iterable[Optional[Mapping[str, Any]]]: """ @@ -159,7 +161,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"projects/{project_gid}/sections" -class Section(AsanaStream): +class Sections(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: section_gid = stream_slice["section_gid"] return f"sections/{section_gid}" From df826ed38299c2f6eaf01b9c2f33504f81484123 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 12:59:05 -0400 Subject: [PATCH 06/11] tests: remove broken unit tests --- .../source-asana/source_asana/oauth.py | 10 ++-- .../source-asana/source_asana/source.py | 3 +- .../source-asana/unit_tests/test_source.py | 41 -------------- .../source-asana/unit_tests/test_streams.py | 53 ------------------- 4 files changed, 7 insertions(+), 100 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-asana/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py index d6c045ac92f8d..c456970b00c52 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py @@ -5,7 +5,7 @@ from typing import Tuple import requests -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator class AsanaOauth2Authenticator(Oauth2Authenticator): @@ -24,13 +24,13 @@ def refresh_access_token(self) -> Tuple[str, int]: Tuple of access token and expiration time in seconds """ data = { - "client_id": (None, self.client_id), - "client_secret": (None, self.client_secret), + "client_id": (None, self._client_id), + "client_secret": (None, self._client_secret), "grant_type": (None, "refresh_token"), - "refresh_token": (None, self.refresh_token), + "refresh_token": (None, self._refresh_token), } - response = requests.post(self.token_refresh_endpoint, files=data) + response = requests.post(self._token_refresh_endpoint, files=data) response.raise_for_status() response_body = response.json() return response_body["access_token"], response_body["expires_in"] diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index e9fb77e0783f2..097ee201c3b42 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -9,7 +9,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator from .streams import CustomFields, Projects, SectionsCompact, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces @@ -19,6 +19,7 @@ class SourceAsana(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: workspaces_stream = Workspaces(authenticator=self._get_authenticator(config)) + logger("THIS IS THE WORKSPACES STREAM", workspaces_stream) next(workspaces_stream.read_records(sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py deleted file mode 100644 index d6353644c9e82..0000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import PropertyMock, patch - -from airbyte_cdk.logger import AirbyteLogger -from source_asana.source import SourceAsana - -logger = AirbyteLogger() - - -def test_check_connection_ok(config, mock_stream, mock_response): - mock_stream("workspaces", response=mock_response) - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert ok - assert not error_msg - - -def test_check_connection_empty_config(config): - config = {} - - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_exception(config): - with patch("source_asana.streams.Workspaces.use_cache", new_callable=PropertyMock, return_value=False): - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_streams(config): - streams = SourceAsana().streams(config) - - assert len(streams) == 11 diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py deleted file mode 100644 index 5113d0fd37e01..0000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -import pytest -import requests_mock as req_mock -from airbyte_cdk.models import SyncMode -from source_asana.streams import AsanaStream, SectionsCompact, Sections, Stories, Tags, Tasks, TeamMemberships, Users - - -@pytest.mark.parametrize( - "stream", - [Tasks, SectionsCompact, Sections, Users, TeamMemberships, Tags, Stories], -) -def test_task_stream(requests_mock, stream, mock_response): - requests_mock.get(req_mock.ANY, json=mock_response) - instance = stream(authenticator=MagicMock()) - - stream_slice = next(instance.stream_slices(sync_mode=SyncMode.full_refresh)) - record = next(instance.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - - assert record - - -@patch.multiple(AsanaStream, __abstractmethods__=set()) -def test_next_page_token(): - stream = AsanaStream() - inputs = {"response": MagicMock()} - expected = "offset" - assert expected in stream.next_page_token(**inputs) - - -@pytest.mark.parametrize( - ("http_status_code", "should_retry"), - [ - (402, False), - (403, False), - (404, False), - (451, False), - (429, True), - ], -) -def test_should_retry(http_status_code, should_retry): - """ - 402, 403, 404, 451 - should not retry. - 429 - should retry. - """ - response_mock = MagicMock() - response_mock.status_code = http_status_code - stream = Stories(MagicMock()) - assert stream.should_retry(response_mock) == should_retry From 461450e4ec663154d5e8f8f992c77665683b57d2 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 13:54:00 -0400 Subject: [PATCH 07/11] fix: remove logger statement --- .../connectors/source-asana/source_asana/source.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 097ee201c3b42..2934e3e766695 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -18,9 +18,10 @@ class SourceAsana(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: - workspaces_stream = Workspaces(authenticator=self._get_authenticator(config)) - logger("THIS IS THE WORKSPACES STREAM", workspaces_stream) - next(workspaces_stream.read_records(sync_mode=SyncMode.full_refresh)) + workspaces_stream = Workspaces( + authenticator=self._get_authenticator(config)) + next(workspaces_stream.read_records( + sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: return False, e From a071d04782f1e17bbd5e2549cb74b75de5f10842 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 13:56:47 -0400 Subject: [PATCH 08/11] format: run black on python files --- .../connectors/source-asana/source_asana/source.py | 6 ++---- .../connectors/source-asana/source_asana/streams.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 2934e3e766695..c16dd8f7885f8 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -18,10 +18,8 @@ class SourceAsana(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: - workspaces_stream = Workspaces( - authenticator=self._get_authenticator(config)) - next(workspaces_stream.read_records( - sync_mode=SyncMode.full_refresh)) + workspaces_stream = Workspaces(authenticator=self._get_authenticator(config)) + next(workspaces_stream.read_records(sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: return False, e diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 90f37f7826d24..19f8f716473c6 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -74,8 +74,7 @@ def get_opt_fields(self) -> MutableMapping[str, str]: if "object" in value["type"]: opt_fields.append(self._handle_object_type(prop, value)) elif "array" in value["type"]: - opt_fields.append(self._handle_array_type( - prop, value.get("items", []))) + opt_fields.append(self._handle_array_type(prop, value.get("items", []))) else: opt_fields.append(prop) From 362f083fde15397ec068a7c834e24908b319afd9 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 18:17:30 +0000 Subject: [PATCH 09/11] Automated Commit - Formatting Changes --- .../connectors/source-asana/source_asana/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index c16dd8f7885f8..72b8f7b245e9e 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator -from .streams import CustomFields, Projects, SectionsCompact, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces +from .streams import CustomFields, Projects, Sections, SectionsCompact, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces class SourceAsana(AbstractSource): From ab394a18748f6d5147104748310536212b1692e4 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 16:54:21 -0400 Subject: [PATCH 10/11] revert: use old auth classes --- .../connectors/source-asana/source_asana/oauth.py | 2 +- .../connectors/source-asana/source_asana/source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py index c456970b00c52..4d22f8873b1e7 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py @@ -5,7 +5,7 @@ from typing import Tuple import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator class AsanaOauth2Authenticator(Oauth2Authenticator): diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 72b8f7b245e9e..977a817cae54e 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -9,7 +9,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator from .streams import CustomFields, Projects, Sections, SectionsCompact, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces From c2470e0d5847a946694cdfce420bb05ac0726c0a Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 16 Oct 2023 18:19:37 -0400 Subject: [PATCH 11/11] fix: add custom parse_response to sections --- .../connectors/source-asana/source_asana/oauth.py | 8 ++++---- .../connectors/source-asana/source_asana/streams.py | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py index 4d22f8873b1e7..d6c045ac92f8d 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/oauth.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/oauth.py @@ -24,13 +24,13 @@ def refresh_access_token(self) -> Tuple[str, int]: Tuple of access token and expiration time in seconds """ data = { - "client_id": (None, self._client_id), - "client_secret": (None, self._client_secret), + "client_id": (None, self.client_id), + "client_secret": (None, self.client_secret), "grant_type": (None, "refresh_token"), - "refresh_token": (None, self._refresh_token), + "refresh_token": (None, self.refresh_token), } - response = requests.post(self._token_refresh_endpoint, files=data) + response = requests.post(self.token_refresh_endpoint, files=data) response.raise_for_status() response_body = response.json() return response_body["access_token"], response_body["expires_in"] diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 19f8f716473c6..b5d8b3b781982 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -168,6 +168,14 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=SectionsCompact, slice_field="section_gid") + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + class Stories(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: