Source S3: Convert to airbyte-lib (#33937)

airbytehq · Jan 9, 2024 · 2edcfb3 · 2edcfb3
1 parent e7ff2a1
commit 2edcfb3
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 38 deletions.
diff --git a/airbyte-integrations/connectors/source-s3/.coveragerc b/airbyte-integrations/connectors/source-s3/.coveragerc
@@ -3,4 +3,5 @@ omit =
     source_s3/exceptions.py
     source_s3/stream.py
     source_s3/utils.py
-    source_s3/source_files_abstract/source.py
+    source_s3/source_files_abstract/source.py
+    source_s3/run.py
diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py
@@ -3,40 +3,7 @@
 #
 
 
-import sys
-import traceback
-from datetime import datetime
-from typing import List
-
-from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
-from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type
-from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader
-
-
-def get_source(args: List[str]):
-    catalog_path = AirbyteEntrypoint.extract_catalog(args)
-    try:
-        return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor)
-    except Exception:
-        print(
-            AirbyteMessage(
-                type=Type.TRACE,
-                trace=AirbyteTraceMessage(
-                    type=TraceType.ERROR,
-                    emitted_at=int(datetime.now().timestamp() * 1000),
-                    error=AirbyteErrorTraceMessage(
-                        message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
-                        stack_trace=traceback.format_exc(),
-                    ),
-                ),
-            ).json()
-        )
-        return None
-
+from source_s3.run import run
 
 if __name__ == "__main__":
-    _args = sys.argv[1:]
-    source = get_source(_args)
-
-    if source:
-        launch(source, _args)
+    run()
diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
-  dockerImageTag: 4.3.0
+  dockerImageTag: 4.3.1
   dockerRepository: airbyte/source-s3
   documentationUrl: https://docs.airbyte.com/integrations/sources/s3
   githubIssueLabel: source-s3

diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py
@@ -33,4 +33,9 @@
     extras_require={
         "tests": TEST_REQUIREMENTS,
     },
+    entry_points={
+        "console_scripts": [
+            "source-s3=source_s3.run:run",
+        ],
+    },
 )
diff --git a/airbyte-integrations/connectors/source-s3/source_s3/run.py b/airbyte-integrations/connectors/source-s3/source_s3/run.py
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+import traceback
+from datetime import datetime
+from typing import List
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type
+from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader
+
+
+def get_source(args: List[str]):
+    catalog_path = AirbyteEntrypoint.extract_catalog(args)
+    try:
+        return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor)
+    except Exception:
+        print(
+            AirbyteMessage(
+                type=Type.TRACE,
+                trace=AirbyteTraceMessage(
+                    type=TraceType.ERROR,
+                    emitted_at=int(datetime.now().timestamp() * 1000),
+                    error=AirbyteErrorTraceMessage(
+                        message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
+                        stack_trace=traceback.format_exc(),
+                    ),
+                ),
+            ).json()
+        )
+        return None
+
+
+def run():
+    _args = sys.argv[1:]
+    source = get_source(_args)
+
+    if source:
+        launch(source, _args)
diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md
@@ -256,6 +256,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [
 
 | Version | Date       | Pull Request                                                                                                    | Subject                                                                                                              |
 |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------|
+| 4.3.1 | 2024-01-04 | [33937](https://github.com/airbytehq/airbyte/pull/33937) | Prepare for airbyte-lib |
 | 4.3.0   | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411)  | Bump CDK version to auto-set primary key for document file streams and support raw txt files         |
 | 4.2.4   | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187)                                                        | Bump CDK version to hide source-defined primary key                                                                  |
 | 4.2.3   | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608)                                                        | Improve document file type parser                                                                                    |
@@ -334,4 +335,4 @@ To perform the text extraction from PDF and Docx files, the connector uses the [
 | 0.1.3   | 2021-08-04 | [5197](https://github.com/airbytehq/airbyte/pull/5197)                                                          | Fixed bug where sync could hang indefinitely on schema inference                                                     |
 | 0.1.2   | 2021-08-02 | [5135](https://github.com/airbytehq/airbyte/pull/5135)                                                          | Fixed bug in spec so it displays in UI correctly                                                                     |
 | 0.1.1   | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990/commits/ff5f70662c5f84eabc03526cddfcc9d73c58c0f4)         | Fixed documentation url in source definition                                                                         |
-| 0.1.0   | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990)                                                          | Created S3 source connector                                                                                          |
+| 0.1.0   | 2021-07-30 | [4990](https://github.com/airbytehq/airbyte/pull/4990)                                                          | Created S3 source connector                                                                                          |