🎉Source Gitlab: preserve data in records with flattened keys (#22293)

* Use get instead of pop to preserve object in dict * update unit tests and schemas * update schemas, fix tests, update version * update version * update expcted records --------- Co-authored-by: Mal Hancock <mallory@archangelic.space> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com>
airbytehq · May 25, 2023 · 01b1673 · 01b1673
1 parent 9b9809b
commit 01b1673
Show file tree

Hide file tree

Showing 16 changed files with 208 additions and 110 deletions.
diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile
@@ -13,5 +13,6 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
+LABEL io.airbyte.version=1.2.0
 LABEL io.airbyte.version=1.1.1
 LABEL io.airbyte.name=airbyte/source-gitlab
diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml
@@ -22,6 +22,10 @@ acceptance_tests:
       expect_records:
         path: "integration_tests/expected_records.jsonl"
       fail_on_extra_columns: false
+      ignored_fields:
+        jobs:
+        - name: "user"
+          bypass_reason: "User object contains local_time which will be different each time test is run"
     - config_path: "secrets/config_with_ids.json"
       timeout_seconds: 3600
       empty_streams:
@@ -32,11 +36,19 @@ acceptance_tests:
       expect_records:
         path: "integration_tests/expected_records_with_ids.jsonl"
       fail_on_extra_columns: false
+      ignored_fields:
+        jobs:
+        - name: "user"
+          bypass_reason: "User object contains local_time which will be different each time test is run"
     - config_path: "secrets/config_oauth.json"
       timeout_seconds: 3600
       expect_records:
         path: "integration_tests/expected_records.jsonl"
       fail_on_extra_columns: false
+      ignored_fields:
+        jobs:
+        - name: "user"
+          bypass_reason: "User object contains local_time which will be different each time test is run"
   incremental:
     tests:
     - config_path: "secrets/config_with_ids.json"

diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl
diff --git a/...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl b/...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl
diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 1.1.1
+  dockerImageTag: 1.2.0
   dockerRepository: airbyte/source-gitlab
   githubIssueLabel: source-gitlab
   icon: gitlab.svg

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json
@@ -31,6 +31,10 @@
     },
     "commit_id": {
       "type": ["null", "string"]
+    },
+    "commit": {
+      "type": ["null", "object"],
+      "additionalProperties": true
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json
@@ -38,6 +38,10 @@
         "type": ["null", "string"]
       }
     },
+    "milestone": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "milestone_id": {
       "type": ["null", "integer"]
     },
@@ -47,9 +51,17 @@
         "type": ["null", "integer"]
       }
     },
+    "assignee": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "assignee_id": {
       "type": ["null", "integer"]
     },
+    "author": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "author_id": {
       "type": ["null", "integer"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json
@@ -35,6 +35,10 @@
     "references": {
       "type": ["null", "object"]
     },
+    "author": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "author_id": {
       "type": ["null", "integer"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json
@@ -98,15 +98,31 @@
     "service_desk_reply_to": {
       "type": ["null", "string"]
     },
+    "author": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "author_id": {
       "type": ["null", "integer"]
     },
+    "assignee": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "assignee_id": {
       "type": ["null", "integer"]
     },
+    "closed_by": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "closed_by_id": {
       "type": ["null", "integer"]
     },
+    "milestone": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "milestone_id": {
       "type": ["null", "integer"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json
@@ -57,15 +57,31 @@
     "tag_list": {
       "type": ["null", "array"]
     },
+    "user": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "user_id": {
       "type": ["null", "integer"]
     },
+    "pipeline": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "pipeline_id": {
       "type": ["null", "integer"]
     },
+    "runner": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "runner_id": {
       "type": ["null", "integer"]
     },
+    "commit": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "commit_id": {
       "type": ["null", "string"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json
@@ -126,18 +126,38 @@
     "approvals_before_merge": {
       "type": ["null", "boolean", "string", "object"]
     },
+    "author": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "author_id": {
       "type": ["null", "integer"]
     },
+    "assignee": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "assignee_id": {
       "type": ["null", "integer"]
     },
+    "closed_by": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "closed_by_id": {
       "type": ["null", "integer"]
     },
+    "milestone": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "milestone_id": {
       "type": ["null", "integer"]
     },
+    "merged_by": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "merged_by_id": {
       "type": ["null", "integer"]
     }

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json
@@ -65,9 +65,17 @@
     "_links": {
       "type": ["null", "object"]
     },
+    "author": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "author_id": {
       "type": ["null", "integer"]
     },
+    "commit": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "commit_id": {
       "type": ["null", "string"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json
@@ -25,6 +25,10 @@
     "protected": {
       "type": ["null", "boolean"]
     },
+    "commit": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
     "commit_id": {
       "type": ["null", "string"]
     },

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
@@ -108,7 +108,7 @@ def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = No
         return record
 
     def _flatten_id(self, record: Dict[str, Any], target: str):
-        target_value = record.pop(target, None)
+        target_value = record.get(target, None)
         record[target + "_id"] = target_value.get("id") if target_value else None
 
     def _flatten_list(self, record: Dict[str, Any], target: str):

diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py
@@ -44,14 +44,14 @@ def test_should_retry(mocker, requests_mock):
                 ]
             ),
         ),
-        [{"commit_id": "c_23", "id": "j_1", "pipeline_id": "p_17", "project_id": "p_1", "runner_id": None, "user_id": "u_1"}]
+        [{"commit": {"id": "c_23"}, "commit_id": "c_23", "id": "j_1", "pipeline": {"id": "p_17"}, "pipeline_id": "p_17", "project_id": "p_1", "runner": None, "runner_id": None, "user": {"id": "u_1"}, "user_id": "u_1"}]
     ),
     (
         tags,
         (
             ("/api/v4/projects/p_1/repository/tags", [{"commit": {"id": "c_1"}, "name": "t_1", "target": "ddc89"}],),
         ),
-        [{"commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}]
+        [{"commit": {"id": "c_1"}, "commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}]
     ),
     (
         releases,
@@ -68,7 +68,7 @@ def test_should_retry(mocker, requests_mock):
                 ],
             ),
         ),
-        [{"author_id": "666", "commit_id": "abcd689", "id": "r_1", "milestones": ["m1", "m2"], "project_id": "p_1"}]
+        [{"author": {"id": "666", "name": "John"}, "author_id": "666", "commit": {"id": "abcd689"}, "commit_id": "abcd689", "id": "r_1", "milestones": ["m1", "m2"], "project_id": "p_1"}]
     ),
     (
         merge_request_commits,

diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md
@@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                    |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
+| 1.2.0   | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys |
 | 1.1.1   | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled |
 | 1.1.0   | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema                                   |
 | 1.0.4   | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme                                                      |