From 45514ce10afa27039bc8a849f9d1787d7aa7c0d6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 19 Dec 2023 11:54:37 +0100 Subject: [PATCH] Prepare Google Sheets, Google Drive, Apify dataset and connector templates for airbyte-lib (#33414) --- .../source-python-http-api/main.py.hbs | 9 ++------- .../source-python-http-api/setup.py.hbs | 5 +++++ .../source_{{snakeCase name}}/run.py.hbs | 13 +++++++++++++ .../source-python/main.py.hbs | 9 ++------- .../source-python/setup.py.hbs | 5 +++++ .../source_{{snakeCase name}}/run.py.hbs | 13 +++++++++++++ .../connectors/source-apify-dataset/Dockerfile | 2 +- .../connectors/source-apify-dataset/main.py | 9 ++------- .../source-apify-dataset/metadata.yaml | 2 +- .../connectors/source-apify-dataset/setup.py | 5 +++++ .../source_apify_dataset/run.py | 15 +++++++++++++++ .../connectors/source-google-drive/main.py | 12 ++---------- .../source-google-drive/metadata.yaml | 2 +- .../connectors/source-google-drive/setup.py | 5 +++++ .../source_google_drive/run.py | 17 +++++++++++++++++ .../connectors/source-google-sheets/main.py | 9 ++------- .../source-google-sheets/metadata.yaml | 2 +- .../connectors/source-google-sheets/setup.py | 5 +++++ .../source_google_sheets/run.py | 15 +++++++++++++++ docs/integrations/sources/apify-dataset.md | 1 + docs/integrations/sources/google-drive.md | 1 + docs/integrations/sources/google-sheets.md | 1 + 22 files changed, 115 insertions(+), 42 deletions(-) create mode 100644 airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs create mode 100644 airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs create mode 100644 airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py create mode 100644 airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py create mode 100644 airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py diff --git a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs index dc8ed8df1dc96..202f3973567d7 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index 667a27713662c..8f3eebe3cef16 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 0000000000000..25c9400301f9b --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +def run(): + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-python/main.py.hbs b/airbyte-integrations/connector-templates/source-python/main.py.hbs index dc8ed8df1dc96..202f3973567d7 100644 --- a/airbyte-integrations/connector-templates/source-python/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python/setup.py.hbs b/airbyte-integrations/connector-templates/source-python/setup.py.hbs index 563d13c3708c5..b16123258acbe 100644 --- a/airbyte-integrations/connector-templates/source-python/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 0000000000000..2ac28a7c8471f --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +if __name__ == "__main__": + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile index c51f4d752c986..6d3b4a5de1b74 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile +++ b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile @@ -34,5 +34,5 @@ COPY source_apify_dataset ./source_apify_dataset ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.1.0 +LABEL io.airbyte.version=2.1.1 LABEL io.airbyte.name=airbyte/source-apify-dataset diff --git a/airbyte-integrations/connectors/source-apify-dataset/main.py b/airbyte-integrations/connectors/source-apify-dataset/main.py index d4b0ed78bc710..4ef9d72f02b4e 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/main.py +++ b/airbyte-integrations/connectors/source-apify-dataset/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_apify_dataset import SourceApifyDataset +from source_apify_dataset.run import run if __name__ == "__main__": - source = SourceApifyDataset() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index 6fc16080b9738..f3421f116b826 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 - dockerImageTag: 2.1.0 + dockerImageTag: 2.1.1 dockerRepository: airbyte/source-apify-dataset githubIssueLabel: source-apify-dataset icon: apify.svg diff --git a/airbyte-integrations/connectors/source-apify-dataset/setup.py b/airbyte-integrations/connectors/source-apify-dataset/setup.py index 51b088de172ce..eb3a7db40ab2e 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/setup.py +++ b/airbyte-integrations/connectors/source-apify-dataset/setup.py @@ -20,4 +20,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-apify-dataset=source_apify_dataset.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py new file mode 100644 index 0000000000000..c7488d02985ea --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceApifyDataset + + +def run(): + source = SourceApifyDataset() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-drive/main.py b/airbyte-integrations/connectors/source-google-drive/main.py index 4d051be2ff54d..606e4f7641e8c 100644 --- a/airbyte-integrations/connectors/source-google-drive/main.py +++ b/airbyte-integrations/connectors/source-google-drive/main.py @@ -2,15 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk import AirbyteEntrypoint -from airbyte_cdk.entrypoint import launch -from source_google_drive import SourceGoogleDrive +from source_google_drive.run import run if __name__ == "__main__": - args = sys.argv[1:] - catalog_path = AirbyteEntrypoint.extract_catalog(args) - source = SourceGoogleDrive(catalog_path) - launch(source, args) + run() diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index a815fe04bee12..bf40629baae06 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.5 + dockerImageTag: 0.0.6 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index 3e46ddcc49ca2..1015092ff1c89 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -29,4 +29,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-google-drive=source_google_drive.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py new file mode 100644 index 0000000000000..a5605f39c2b17 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_google_drive import SourceGoogleDrive + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + source = SourceGoogleDrive(catalog_path) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-google-sheets/main.py b/airbyte-integrations/connectors/source-google-sheets/main.py index 4aaa9a106d9bf..806ac60fbefe5 100644 --- a/airbyte-integrations/connectors/source-google-sheets/main.py +++ b/airbyte-integrations/connectors/source-google-sheets/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_sheets import SourceGoogleSheets +from source_google_sheets.run import run if __name__ == "__main__": - source = SourceGoogleSheets() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml index eaf059bbbcf9b..2b937318f2dbb 100644 --- a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 - dockerImageTag: 0.3.11 + dockerImageTag: 0.3.12 dockerRepository: airbyte/source-google-sheets documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets githubIssueLabel: source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 1dd377a9357ef..921d19138dbe2 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -33,4 +33,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-google-sheets=source_google_sheets.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py new file mode 100644 index 0000000000000..a34dfe611d013 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceGoogleSheets + + +def run(): + source = SourceGoogleSheets() + launch(source, sys.argv[1:]) diff --git a/docs/integrations/sources/apify-dataset.md b/docs/integrations/sources/apify-dataset.md index 7370483c115f0..a6546160709d9 100644 --- a/docs/integrations/sources/apify-dataset.md +++ b/docs/integrations/sources/apify-dataset.md @@ -69,6 +69,7 @@ The Apify dataset connector uses [Apify Python Client](https://docs.apify.com/ap | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------------- | :-------------------------------------------------------------------------- | +| 2.1.1 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 2.1.0 | 2023-10-13 | [31333](https://github.com/airbytehq/airbyte/pull/31333) | Add stream for arbitrary datasets | | 2.0.0 | 2023-09-18 | [30428](https://github.com/airbytehq/airbyte/pull/30428) | Fix broken stream, manifest refactor | | 1.0.0 | 2023-08-25 | [29859](https://github.com/airbytehq/airbyte/pull/29859) | Migrate to lowcode | diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index fcaa57bdfd16b..cc0a92099b4ef 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -247,6 +247,7 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|-----------------------------------------------------------|--------------------------------------------------------------| +| 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 0.0.5 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | | 0.0.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.0.3 | 2023-11-16 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Improve folder id input and update document file type parser | diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index 4fb8abb013c27..88e12469eee55 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -151,6 +151,7 @@ Airbyte batches requests to the API in order to efficiently pull data and respec | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| +| 0.3.12 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | | 0.3.11 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.10 | 2023-09-27 | [30487](https://github.com/airbytehq/airbyte/pull/30487) | Fix bug causing rows to be skipped when batch size increased due to rate limits. | | 0.3.9 | 2023-09-25 | [30749](https://github.com/airbytehq/airbyte/pull/30749) | Performance testing - include socat binary in docker image |