diff --git a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md index 2fc7a562ca16a..9be462491a184 100644 --- a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md +++ b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md @@ -6,7 +6,7 @@ Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-de Some streams depend on: - workspaces (Teams, Users, CustomFields, Projects, Tags, Users streams); -- projects (Sections, Tasks streams); +- projects (SectionsCompact, Sections, Tasks streams); - tasks (Stories stream); - teams (TeamMemberships stream). diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index 07ff709fff050..5fd80f2605522 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json index 104d114b919af..3e9a2f61f9ecf 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json @@ -9,6 +9,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sections_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sections", diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 00a96f584b23b..1c6e0f19fe188 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/source-asana documentationUrl: https://docs.airbyte.com/integrations/sources/asana githubIssueLabel: source-asana diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json new file mode 100644 index 0000000000000..39d0b95b5bf84 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json @@ -0,0 +1,14 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 7c781fc1e0bf7..977a817cae54e 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator -from .streams import CustomFields, Projects, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces +from .streams import CustomFields, Projects, Sections, SectionsCompact, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces class SourceAsana(AbstractSource): @@ -46,6 +46,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ CustomFields(**args), Projects(**args), + SectionsCompact(**args), Sections(**args), Stories(**args), Tags(**args), diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 081d2a140ed92..b5d8b3b781982 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -91,7 +91,8 @@ def _handle_array_type(self, prop: str, value: MutableMapping[str, Any]) -> str: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() - yield from response_json.get("data", []) # Asana puts records in a container array "data" + # Asana puts records in a container array "data" + yield from response_json.get("data", []) def read_slices_from_records(self, stream_class: AsanaStreamType, slice_field: str) -> Iterable[Optional[Mapping[str, Any]]]: """ @@ -132,7 +133,7 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Mu class ProjectRelatedStream(AsanaStream, ABC): """ - Few streams (Sections and Tasks) depends on `project gid`: Sections as a part of url and Tasks as `projects` + Few streams (SectionsCompact and Tasks) depends on `project gid`: SectionsCompact as a part of url and Tasks as `projects` argument in request. """ @@ -153,12 +154,29 @@ def path(self, **kwargs) -> str: return "projects" -class Sections(ProjectRelatedStream): +class SectionsCompact(ProjectRelatedStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: project_gid = stream_slice["project_gid"] return f"projects/{project_gid}/sections" +class Sections(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + section_gid = stream_slice["section_gid"] + return f"sections/{section_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=SectionsCompact, slice_field="section_gid") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + + class Stories(AsanaStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: task_gid = stream_slice["task_gid"] diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py deleted file mode 100644 index 7838cd23b8d56..0000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import PropertyMock, patch - -from airbyte_cdk.logger import AirbyteLogger -from source_asana.source import SourceAsana - -logger = AirbyteLogger() - - -def test_check_connection_ok(config, mock_stream, mock_response): - mock_stream("workspaces", response=mock_response) - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert ok - assert not error_msg - - -def test_check_connection_empty_config(config): - config = {} - - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_exception(config): - with patch("source_asana.streams.Workspaces.use_cache", new_callable=PropertyMock, return_value=False): - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_streams(config): - streams = SourceAsana().streams(config) - - assert len(streams) == 10 diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py deleted file mode 100644 index 377a2e3f5181e..0000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -import pytest -import requests_mock as req_mock -from airbyte_cdk.models import SyncMode -from source_asana.streams import AsanaStream, Sections, Stories, Tags, Tasks, TeamMemberships, Users - - -@pytest.mark.parametrize( - "stream", - [Tasks, Sections, Users, TeamMemberships, Tags, Stories], -) -def test_task_stream(requests_mock, stream, mock_response): - requests_mock.get(req_mock.ANY, json=mock_response) - instance = stream(authenticator=MagicMock()) - - stream_slice = next(instance.stream_slices(sync_mode=SyncMode.full_refresh)) - record = next(instance.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - - assert record - - -@patch.multiple(AsanaStream, __abstractmethods__=set()) -def test_next_page_token(): - stream = AsanaStream() - inputs = {"response": MagicMock()} - expected = "offset" - assert expected in stream.next_page_token(**inputs) - - -@pytest.mark.parametrize( - ("http_status_code", "should_retry"), - [ - (402, False), - (403, False), - (404, False), - (451, False), - (429, True), - ], -) -def test_should_retry(http_status_code, should_retry): - """ - 402, 403, 404, 451 - should not retry. - 429 - should retry. - """ - response_mock = MagicMock() - response_mock.status_code = http_status_code - stream = Stories(MagicMock()) - assert stream.should_retry(response_mock) == should_retry diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index dede4313e1a3e..0d9b94d7e6b63 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -68,6 +68,7 @@ The connector is restricted by normal Asana [requests limitation](https://develo | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.8 | 2023-10-16 | [31009](https://github.com/airbytehq/airbyte/pull/31009) | Add SectionsCompact stream | | 0.1.7 | 2023-05-29 | [26716](https://github.com/airbytehq/airbyte/pull/26716) | Remove authSpecification from spec.json, use advancedAuth instead | | 0.1.6 | 2023-05-26 | [26653](https://github.com/airbytehq/airbyte/pull/26653) | Fix order of authentication methods | | 0.1.5 | 2022-11-16 | [19561](https://github.com/airbytehq/airbyte/pull/19561) | Added errors handling, updated SAT with new format |