🐛 Source Github: add incremental for repositories, workflows, `pu…

…ll_request_comment_reactions`, `issue_reactions`, `issue_comment_reactions`, `commit_comment_reactions` (#12294) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · May 23, 2022 · e215822 · e215822
1 parent 9c423b7
commit e215822
Show file tree

Hide file tree

Showing 16 changed files with 300 additions and 70 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -295,7 +295,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.29
+  dockerImageTag: 0.2.30
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2503,7 +2503,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.2.29"
+- dockerImage: "airbyte/source-github:0.2.30"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.29
+LABEL io.airbyte.version=0.2.30
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml
@@ -21,24 +21,30 @@ tests:
       future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
         comments: ["airbytehq/integration-test", "updated_at"]
+        commit_comment_reactions: ["airbytehq/integration-test", "55538825", "created_at"]
         commit_comments: ["airbytehq/integration-test", "updated_at"]
         commits: ["airbytehq/integration-test", "master", "created_at"]
         deployments: ["airbytehq/integration-test", "updated_at"]
         events: ["airbytehq/integration-test", "created_at"]
+        issue_comment_reactions: ["airbytehq/integration-test", "907296275", "created_at"]
         issue_events: ["airbytehq/integration-test", "created_at"]
         issue_milestones: ["airbytehq/integration-test", "updated_at"]
+        issue_reactions: ["airbytehq/integration-test", "11", "created_at"]
         issues: ["airbytehq/integration-test", "updated_at"]
         project_cards:
           ["airbytehq/integration-test", "13167124", "17807006", "updated_at"]
         project_columns:
           ["airbytehq/integration-test", "13167124", "updated_at"]
         projects: ["airbytehq/integration-test", "updated_at"]
+        pull_request_comment_reactions: ["airbytehq/integration-test", "699253726", "created_at"]
         pull_request_stats: ["airbytehq/integration-test", "updated_at"]
         pull_requests: ["airbytehq/integration-test", "updated_at"]
         releases: ["airbytehq/integration-test", "created_at"]
+        repositories: ["airbytehq", "updated_at"]
         review_comments: ["airbytehq/integration-test", "updated_at"]
         reviews: ["airbytehq/integration-test", "pull_request_updated_at"]
         stargazers: ["airbytehq/integration-test", "starred_at"]
+        workflows: ["airbytehq/integration-test", "updated_at"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json
@@ -4,6 +4,16 @@
       "updated_at": "2121-06-30T10:22:10Z"
     }
   },
+  "commit_comment_reactions": {
+    "airbytehq/integration-test": {
+      "55538825": {
+        "created_at": "2121-12-31T23:59:59Z"
+      },
+      "55538840": {
+        "created_at": "2121-12-31T23:59:59Z"
+      }
+    }
+  },
   "commit_comments": {
     "airbytehq/integration-test": {
       "updated_at": "2121-04-30T20:36:17Z"
@@ -24,6 +34,13 @@
       "created_at": "2121-06-29T03:44:45Z"
     }
   },
+  "issue_comment_reactions": {
+    "airbytehq/integration-test": {
+      "907296275": {
+        "created_at": "2121-12-31T23:59:59Z"
+      }
+    }
+  },
   "issue_events": {
     "airbytehq/integration-test": {
       "created_at": "2121-06-29T01:49:42Z"
@@ -34,6 +51,13 @@
       "updated_at": "2121-06-25T22:28:33Z"
     }
   },
+  "issue_reactions": {
+    "airbytehq/integration-test": {
+      "11": {
+        "created_at": "2121-12-31T23:59:59Z"
+      }
+    }
+  },
   "issues": {
     "airbytehq/integration-test": {
       "updated_at": "2121-06-30T06:44:42Z"
@@ -63,6 +87,13 @@
       "updated_at": "2121-06-28T17:24:51Z"
     }
   },
+  "pull_request_comment_reactions": {
+    "airbytehq/integration-test": {
+      "699253726": {
+        "created_at": "2121-12-31T23:59:59Z"
+      }
+    }
+  },
   "pull_request_stats": {
     "airbytehq/integration-test": {
       "updated_at": "2121-06-29T02:04:57Z"
@@ -78,6 +109,11 @@
       "created_at": "2121-06-23T23:57:07Z"
     }
   },
+  "repositories": {
+    "airbytehq": {
+      "updated_at": "2121-12-31T23:59:59Z"
+    }
+  },
   "review_comments": {
     "airbytehq/integration-test": {
       "updated_at": "2121-06-23T23:57:07Z"
@@ -92,5 +128,10 @@
     "airbytehq/integration-test": {
       "starred_at": "2121-06-29T02:04:57Z"
     }
+  },
+  "workflows": {
+    "airbytehq/integration-test": {
+      "updated_at": "2121-12-31T23:59:59Z"
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json
@@ -47,10 +47,10 @@
       "stream": {
         "name": "commit_comment_reactions",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -109,10 +109,10 @@
       "stream": {
         "name": "issue_comment_reactions",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -155,10 +155,10 @@
       "stream": {
         "name": "issue_reactions",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -227,10 +227,10 @@
       "stream": {
         "name": "pull_request_comment_reactions",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -286,10 +286,10 @@
       "stream": {
         "name": "repositories",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -365,10 +365,10 @@
       "stream": {
         "name": "workflows",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
     {

diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py
@@ -8,6 +8,7 @@
 MAIN_REQUIREMENTS = [
     "airbyte-cdk~=0.1.33",
     "vcrpy==4.1.1",
+    "pendulum~=2.1.2",
 ]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "responses~=0.19.0"]

diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_reactions.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_reactions.json
@@ -1,4 +1,28 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "$ref": "reaction.json"
+  "type": ["null", "object"],
+  "properties": {
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "node_id": {
+      "type": ["null", "string"]
+    },
+    "content": {
+      "type": ["null", "string"]
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "user": {
+      "$ref": "user.json"
+    },
+    "repository": {
+      "type": "string"
+    },
+    "issue_number": {
+      "type": "integer"
+    }
+  }
 }
diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json b/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json
@@ -244,7 +244,7 @@
       "format": "date-time"
     },
     "updated_at": {
-      "type": ["null", "string"],
+      "type": "string",
       "format": "date-time"
     },
     "permissions": {

diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/reaction.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/reaction.json
@@ -11,14 +11,17 @@
       "type": ["null", "string"]
     },
     "created_at": {
-      "type": ["null", "string"],
+      "type": "string",
       "format": "date-time"
     },
     "user": {
       "$ref": "user.json"
     },
     "repository": {
-      "type": ["null", "string"]
+      "type": "string"
+    },
+    "comment_id": {
+      "type": "integer"
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/workflows.json b/airbyte-integrations/connectors/source-github/source_github/schemas/workflows.json
@@ -18,10 +18,12 @@
       "type": ["null", "string"]
     },
     "created_at": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "updated_at": {
-      "type": ["null", "string"]
+      "type": "string",
+      "format": "date-time"
     },
     "url": {
       "type": ["null", "string"]
@@ -31,6 +33,9 @@
     },
     "badge_url": {
       "type": ["null", "string"]
+    },
+    "repository": {
+      "type": "string"
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py
@@ -179,6 +179,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         page_size = config.get("page_size_for_large_streams", DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM)
 
         organization_args = {"authenticator": authenticator, "organizations": organizations}
+        organization_args_with_start_date = {**organization_args, "start_date": config["start_date"]}
         repository_args = {"authenticator": authenticator, "repositories": repositories, "page_size_for_large_streams": page_size}
         repository_args_with_start_date = {**repository_args, "start_date": config["start_date"]}
 
@@ -214,7 +215,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             PullRequestStats(parent=pull_requests_stream, **repository_args_with_start_date),
             pull_requests_stream,
             Releases(**repository_args_with_start_date),
-            Repositories(**organization_args),
+            Repositories(**organization_args_with_start_date),
             ReviewComments(**repository_args_with_start_date),
             Reviews(parent=pull_requests_stream, **repository_args_with_start_date),
             Stargazers(**repository_args_with_start_date),