From eeccca709ba54ff60f768b931319ba4bb55c10b7 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Thu, 26 May 2022 19:58:45 -0700 Subject: [PATCH 1/5] GitLab Source: add GroupIssueBoards stream --- .../integration_tests/configured_catalog.json | 10 +++ .../schemas/group_issue_boards.json | 63 +++++++++++++++++++ .../source-gitlab/source_gitlab/source.py | 2 + .../source-gitlab/source_gitlab/streams.py | 5 ++ 4 files changed, 80 insertions(+) create mode 100644 airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json index 5cbafa2a2dd26..5c0d4dc8c29aa 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json @@ -43,6 +43,16 @@ "destination_sync_mode": "append", "cursor_field": ["created_at"] }, + { + "stream": { + "name": "group_issue_boards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "issues", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json new file mode 100644 index 0000000000000..6195d2f6ea935 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "hide_backlog_list": { + "type": ["null", "boolean"] + }, + "hide_closed_list": { + "type": ["null", "boolean"] + }, + "project": { + "type": ["null", "integer"] + }, + "lists": { + "type": ["null", "array"], + "items": [ + { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "description_html": { + "type": ["null", "string"] + }, + "text_color": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + } + } + }, + "position": { + "type": ["null", "integer"] + } + } + } + ] + }, + "group_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index 12db841307487..ca162994301a3 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -17,6 +17,7 @@ EpicIssues, Epics, GitlabStream, + GroupIssueBoards, GroupLabels, GroupMembers, GroupMilestones, @@ -105,6 +106,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Commits(parent_stream=projects, repository_part=True, start_date=config["start_date"], **auth_params), epics, EpicIssues(parent_stream=epics, **auth_params), + GroupIssueBoards(parent_stream=groups, **auth_params), Issues(parent_stream=projects, start_date=config["start_date"], **auth_params), Jobs(parent_stream=pipelines, **auth_params), ProjectMilestones(parent_stream=projects, **auth_params), diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index e134a2075acbc..114d56716ea1b 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -306,6 +306,11 @@ def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): return record +class GroupIssueBoards(GitlabChildStream): + path_template = "groups/{id}/boards" + flatten_parent_id = True + + class Users(GitlabChildStream): pass From a9b8e842e6b658045a612aeceae60e646375bec7 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Tue, 31 May 2022 11:07:49 -0700 Subject: [PATCH 2/5] Address stream schema comments --- .../source_gitlab/schemas/group_issue_boards.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json index 6195d2f6ea935..3422f8aaeec0b 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json @@ -21,13 +21,13 @@ "type": ["null", "array"], "items": [ { - "type": "object", + "type": ["null", "object"], "properties": { "id": { "type": ["null", "integer"] }, "label": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { "type": ["null", "integer"] From 1576178b2ec3b0f35b4f34309e1858208233a465 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Tue, 31 May 2022 18:34:53 -0700 Subject: [PATCH 3/5] Address comments --- .../schemas/group_issue_boards.json | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json index 3422f8aaeec0b..4d50781a45cc7 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json @@ -19,42 +19,40 @@ }, "lists": { "type": ["null", "array"], - "items": [ - { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "description_html": { - "type": ["null", "string"] - }, - "text_color": { - "type": ["null", "string"] - }, - "color": { - "type": ["null", "string"] - } + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "description_html": { + "type": ["null", "string"] + }, + "text_color": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] } - }, - "position": { - "type": ["null", "integer"] } + }, + "position": { + "type": ["null", "integer"] } } - ] + } }, "group_id": { "type": ["null", "integer"] From 3acd0093327fbae4f8206337f12f571c20fff955 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Thu, 23 Jun 2022 10:18:56 -0700 Subject: [PATCH 4/5] Bump version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-gitlab/Dockerfile | 2 +- docs/integrations/sources/gitlab.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a4a443efc2444..e670eed9f44e6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -311,7 +311,7 @@ - name: Gitlab sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 dockerRepository: airbyte/source-gitlab - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab icon: gitlab.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index f831f3760d4c6..68047e54ba2de 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index ce17f977fe812..b01a0e598f703 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -63,6 +63,7 @@ GitLab source is working with GitLab API v4. It can also work with self-hosted G | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------| :--- | +| 0.1.6 | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream | | 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | | 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | | 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | From f09ffea0ddbf8c3d12ffee654e1bc9a37e87eeea Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Mon, 27 Jun 2022 09:53:01 -0700 Subject: [PATCH 5/5] Add as empty stream --- .../connectors/source-gitlab/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml index e0be76c660f70..ff6869c4e821a 100644 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml @@ -12,6 +12,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["group_issue_boards"] # We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately, # and the Acceptance Tests at this stage do not support this functionality. # Therefore, we hardcode the cursor_paths for our config.