From 4e049ccda6c25f326e2a14c89dd8ab7b8066e70a Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Thu, 5 Oct 2023 01:21:18 +0100 Subject: [PATCH 1/3] add attachmentscompact and attachments streams --- .../integration_tests/configured_catalog.json | 18 +++++ .../source_asana/schemas/attachments.json | 67 +++++++++++++++++++ .../schemas/attachments_compact.json | 17 +++++ .../source-asana/source_asana/source.py | 4 +- .../source-asana/source_asana/streams.py | 24 +++++++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json create mode 100644 airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json index 104d114b919af..18574410eaeb6 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json @@ -1,5 +1,23 @@ { "streams": [ + { + "stream": { + "name": "attachments_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attachments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "projects", diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json new file mode 100644 index 0000000000000..e12e779c518f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json @@ -0,0 +1,67 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "download_url": { + "type": ["null", "string"] + }, + "permanent_url": { + "type": ["null", "string"] + }, + "host": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + }, + "created_by": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + } + } + } + } + }, + "size": { + "type": ["null", "integer"] + }, + "view_url": { + "type": ["null", "string"] + }, + "connected_to_app": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json new file mode 100644 index 0000000000000..cf9295d85c583 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json @@ -0,0 +1,17 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 7c781fc1e0bf7..e7b8e7f8d13a6 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator -from .streams import CustomFields, Projects, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces +from .streams import AttachmentsCompact, Attachments, CustomFields, Projects, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces class SourceAsana(AbstractSource): @@ -44,6 +44,8 @@ def _get_authenticator(config: dict) -> Union[TokenAuthenticator, AsanaOauth2Aut def streams(self, config: Mapping[str, Any]) -> List[Stream]: args = {"authenticator": self._get_authenticator(config)} return [ + AttachmentsCompact(**args), + Attachments(**args), CustomFields(**args), Projects(**args), Sections(**args), diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 081d2a140ed92..33534fc40eafe 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -139,6 +139,30 @@ class ProjectRelatedStream(AsanaStream, ABC): def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=Projects, slice_field="project_gid") +class ProjectBriefs(WorkspaceRequestParamsRelatedStream): + def path(self, **kwargs) -> str: + return "projects" + +class AttachmentsCompact(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "attachments" + + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["parent"] = stream_slice["parent_gid"] + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=Projects, slice_field="parent_gid") + yield from self.read_slices_from_records(stream_class=Tasks, slice_field="parent_gid") + +class Attachments(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + attachment_gid = stream_slice["attachment_gid"] + return f"attachments/{attachment_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=AttachmentsCompact, slice_field="attachment_gid") class CustomFields(WorkspaceRelatedStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: From cd9e99c162a38b12b03b3a6bdc768b532be3e765 Mon Sep 17 00:00:00 2001 From: sajarin Date: Tue, 17 Oct 2023 16:45:59 -0400 Subject: [PATCH 2/3] docs: bump versions, add changelog entry, format files --- airbyte-integrations/connectors/source-asana/Dockerfile | 2 +- airbyte-integrations/connectors/source-asana/metadata.yaml | 2 +- docs/integrations/sources/asana.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index 3875e6c1e6405..1aa0e4825aab9 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 9c3d63d9a5706..84ca331b718b2 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf - dockerImageTag: 0.1.9 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-asana documentationUrl: https://docs.airbyte.com/integrations/sources/asana githubIssueLabel: source-asana diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index cd3b4ec3ae575..6c8eab08670b7 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -68,6 +68,7 @@ The connector is restricted by normal Asana [requests limitation](https://develo | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.2.0 | 2023-10-17 | [31090](https://github.com/airbytehq/airbyte/pull/31090) | Add Attachments stream | | 0.1.9 | 2023-10-16 | [31089](https://github.com/airbytehq/airbyte/pull/31089) | Add Events stream | | 0.1.8 | 2023-10-16 | [31009](https://github.com/airbytehq/airbyte/pull/31009) | Add SectionsCompact stream | | 0.1.7 | 2023-05-29 | [26716](https://github.com/airbytehq/airbyte/pull/26716) | Remove authSpecification from spec.json, use advancedAuth instead | From 2be114d0f98cac4dc2ea4f3804a0f95aa93e81f3 Mon Sep 17 00:00:00 2001 From: sajarin Date: Thu, 19 Oct 2023 14:35:45 -0400 Subject: [PATCH 3/3] fix: add caching and format files --- .../connectors/source-asana/source_asana/source.py | 2 +- .../connectors/source-asana/source_asana/streams.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index d67f02fcb4feb..d2f1943b94233 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -13,8 +13,8 @@ from source_asana.oauth import AsanaOauth2Authenticator from .streams import ( - AttachmentsCompact, Attachments, + AttachmentsCompact, CustomFields, Projects, Sections, diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 3d40785376ec9..2ddf40bb2b3f6 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -147,6 +147,8 @@ def path(self, **kwargs) -> str: class AttachmentsCompact(AsanaStream): + use_cache = True + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return "attachments" @@ -168,6 +170,14 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=AttachmentsCompact, slice_field="attachment_gid") + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + class CustomFields(WorkspaceRelatedStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -240,6 +250,8 @@ def path(self, **kwargs) -> str: class Tasks(ProjectRelatedStream): + use_cache = True + def path(self, **kwargs) -> str: return "tasks"