From 14f778ce50928fde18097fadde04c76c8a0e9a8c Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Mon, 3 Oct 2022 18:34:31 -0400 Subject: [PATCH 01/32] fixed link --- docs/integrations/sources/paypal-transaction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index 7fada53271caca..d72e5dc168b715 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -13,7 +13,7 @@ In order to get an `Client ID` and `Secret` please go to [this](https://develope :::note -Note: Our Paypal Transactions Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Paypal Transactions is critical to your business, [please reach out to us](product@airbyte.io) to discuss how we may be able to partner on this effort. +Note: Our Paypal Transactions Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Paypal Transactions is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort. ::: From 11c4e93dffed76c9557ae73373cb832dec2b390e Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Mon, 3 Oct 2022 18:40:19 -0400 Subject: [PATCH 02/32] Added redirect --- docusaurus/docusaurus.config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index b337880e0f1b53..fbbf82de3d0d63 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -49,6 +49,10 @@ const config = { from: '/operator-guides/securing-airbyte', to: '/operator-guides/security', }, + { + from: '/connector-development/config-based/', + to: '/connector-development/config-based/low-code-cdk-overview', + }, // { // from: '/some-lame-path', // to: '/a-much-cooler-uri', From f94185efc9bd641d9136aa879af44410c9d5d385 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Tue, 4 Oct 2022 01:00:06 +0200 Subject: [PATCH 03/32] =?UTF-8?q?=F0=9F=90=9BSource=20Amplitude:=20fix=20s?= =?UTF-8?q?tart=20date=20validation=20(#17320)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix start date validation * Add unittests * Updated to review * Delteed print * Updated version * Updated version * Updated to SAT * Updated to unittest * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-amplitude/Dockerfile | 2 +- .../integration_tests/configured_catalog.json | 46 ------------------- .../source-amplitude/source_amplitude/api.py | 4 +- .../source_amplitude/source.py | 16 +++++++ .../unit_tests/test_source.py | 8 ++++ docs/integrations/sources/amplitude.md | 3 +- 8 files changed, 32 insertions(+), 51 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 115fb24469c007..8ef17fdd8f27c6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -56,7 +56,7 @@ - name: Amplitude sourceDefinitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 dockerRepository: airbyte/source-amplitude - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/sources/amplitude icon: amplitude.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 25a29d0d174e2e..4215500efbf2e9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1105,7 +1105,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amplitude:0.1.14" +- dockerImage: "airbyte/source-amplitude:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/amplitude" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile index b58dea27445d0a..efe89edf1650aa 100644 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json index 5a9f565cb48deb..6104126764c2cf 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json @@ -1,38 +1,5 @@ { "streams": [ - { - "stream": { - "name": "cohorts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "annotations", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["event_time"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["event_time"] - }, { "stream": { "name": "active_users", @@ -45,19 +12,6 @@ "sync_mode": "incremental", "destination_sync_mode": "append", "cursor_field": ["date"] - }, - { - "stream": { - "name": "average_session_length", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["date"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["date"] } ] } diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py index bf19237f1429c8..544c7d6b0d9479 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py @@ -60,7 +60,7 @@ class IncrementalAmplitudeStream(AmplitudeStream, ABC): def __init__(self, start_date: str, **kwargs): super().__init__(**kwargs) - self._start_date = pendulum.parse(start_date) + self._start_date = pendulum.parse(start_date) if isinstance(start_date, str) else start_date @property @abstractmethod @@ -131,6 +131,7 @@ def request_params( "end": self._get_end_date(start_datetime).strftime(self.date_template), } ) + return params @@ -177,6 +178,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite } ) start = start.add(**self.time_interval) + return slices def read_records( diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py index 51552b3e903675..39315fadbb83b6 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py @@ -2,9 +2,11 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import logging from base64 import b64encode from typing import Any, List, Mapping, Tuple +import pendulum from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource @@ -29,10 +31,24 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> except Exception as error: return False, f"Unable to connect to Amplitude API with the provided credentials - {repr(error)}" + @staticmethod + def _validate_start_date(start_date): + now = pendulum.now() + start_date = pendulum.parse(start_date) + start_date_in_future = start_date > now + + if start_date_in_future: + logger = logging.getLogger("airbyte") + logger.info(f"Start date set to {now}.") + + return now + return start_date + def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ + config["start_date"] = self._validate_start_date(config.get("start_date")) auth = TokenAuthenticator(token=self._convert_auth_to_token(config["api_key"], config["secret_key"]), auth_method="Basic") return [ diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_source.py index f1a7731cc4016a..0c8347b3291af5 100644 --- a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_source.py @@ -5,6 +5,7 @@ from unittest.mock import patch +import pendulum import pytest import requests from airbyte_cdk import AirbyteLogger @@ -53,7 +54,14 @@ def test_check(response, check_passed): ids=["Cohorts", "Annotations", "ActiveUsers", "AverageSessionLength", "Events"], ) def test_streams(expected_stream_cls): + TEST_CONFIG["start_date"] = pendulum.tomorrow().to_datetime_string() streams = TEST_INSTANCE.streams(config=TEST_CONFIG) for stream in streams: if expected_stream_cls in streams: assert isinstance(stream, expected_stream_cls) + + +def test_validate_start_date(): + start_date = pendulum.tomorrow().to_datetime_string() + now = pendulum.now().to_datetime_string() + assert TEST_INSTANCE._validate_start_date(start_date).to_datetime_string() == now diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md index 01b8e0c4df4acf..3be515e1a1e0e0 100644 --- a/docs/integrations/sources/amplitude.md +++ b/docs/integrations/sources/amplitude.md @@ -42,7 +42,8 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 0.1.15 | 2022-10-03 | [17320](https://github.com/airbytehq/airbyte/pull/17320) | Add validation `start_date` filed if it's in the future | | 0.1.14 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.1.13 | 2022-08-31 | [16185](https://github.com/airbytehq/airbyte/pull/16185) | Re-release on new `airbyte_cdk==0.1.81` | | 0.1.12 | 2022-08-11 | [15506](https://github.com/airbytehq/airbyte/pull/15506) | Changed slice day window to 1, instead of 3 for Events stream | From 06186d2596e6f9e83acf901ebd33ddb42d5a501d Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Tue, 4 Oct 2022 01:07:36 +0200 Subject: [PATCH 04/32] =?UTF-8?q?=F0=9F=90=9BSource=20Hubspot:=20Fixed=204?= =?UTF-8?q?03=20error=20validation=20(#16914)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed 403 error validation * Updated PR number * Updated after review * Updated after review * Updated version * Add ignoredd fields * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../connectors/source-hubspot/README.md | 2 +- .../source-hubspot/acceptance-test-config.yml | 2 ++ .../source-hubspot/source_hubspot/streams.py | 7 ++++++ .../source-hubspot/unit_tests/test_source.py | 23 ++++++++----------- docs/integrations/sources/hubspot.md | 3 ++- 8 files changed, 25 insertions(+), 18 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8ef17fdd8f27c6..26a31ff513a5ac 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -472,7 +472,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 4215500efbf2e9..d7a77544239750 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4715,7 +4715,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.2.1" +- dockerImage: "airbyte/source-hubspot:0.2.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 86a7233f7bbbe7..fcb67bf2bd5fa5 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index ca97836b945b43..3d70b60a467ceb 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -99,7 +99,7 @@ python main.py read --config secrets/config.json --catalog sample_files/configur Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install .'[tests]' ``` ### Unit Tests diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index ea890d370b601e..ac1d4738c9e240 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -40,6 +40,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/full_refresh_catalog.json" ignored_fields: + "contact_lists": [ "properties", "ilsFilterBranch" ] "companies": [ "properties", "hs_time_in_customer" ] "companies": [ "properties", "hs_time_in_evangelist" ] "companies": [ "properties", "hs_time_in_lead" ] @@ -73,6 +74,7 @@ tests: - config_path: "secrets/config_oauth.json" configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json" ignored_fields: + "contact_lists": [ "properties", "ilsFilterBranch" ] "companies": [ "properties", "hs_time_in_customer" ] "companies": [ "properties", "hs_time_in_evangelist" ] "companies": [ "properties", "hs_time_in_lead" ] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 999dd7308bd51a..8a664c8f6c498d 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -208,6 +208,7 @@ class Stream(HttpStream, ABC): primary_key = None filter_old_records: bool = True denormalize_records: bool = False # one record from API response can result in multiple records emitted + raise_on_http_errors: bool = True @property @abstractmethod @@ -259,6 +260,12 @@ def __init__(self, api: API, start_date: Union[str, pendulum.datetime], credenti elif creds_title in (OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS): self._authenticator = api.get_authenticator() + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == HTTPStatus.FORBIDDEN: + setattr(self, "raise_on_http_errors", False) + logger.warning("You have not permission to API for this stream. " "Please check your scopes for Hubspot account.") + return super().should_retry(response) + def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == codes.too_many_requests: return float(response.headers.get("Retry-After", 3)) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 8e9949c9416565..1adb922370c1f9 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -142,15 +142,11 @@ def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions, """ # Mapping tipical response for mocker - responses = [ - { - "json": { - "status": "error", - "message": f'This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])', - "correlationId": "2fe0a9af-3609-45c9-a4d7-83a1774121aa", - } - } - ] + json = { + "status": "error", + "message": f'This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])', + "correlationId": "2fe0a9af-3609-45c9-a4d7-83a1774121aa", + } # We expect something like this expected_warining_message = { @@ -165,11 +161,12 @@ def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions, test_stream = Workflows(**common_params) # Mocking Request - requests_mock.register_uri("GET", test_stream.url, responses) - list(test_stream.read_records(sync_mode=SyncMode.full_refresh)) + requests_mock.register_uri("GET", test_stream.url, json=json, status_code=403) + records = list(test_stream.read_records(sync_mode=SyncMode.full_refresh)) # match logged expected logged warning message with output given from preudo-output assert expected_warining_message["log"]["message"] in caplog.text + assert not records class TestSplittingPropertiesFunctionality: @@ -354,7 +351,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-02-25T16:43:11Z"} for y in range(100)], "paging": { "next": { - "after": f"{x*100}", + "after": f"{x * 100}", } }, }, @@ -370,7 +367,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req "results": [{"id": f"{y}", "updatedAt": "2022-03-01T00:00:00Z"} for y in range(100)], "paging": { "next": { - "after": f"{x*100}", + "after": f"{x * 100}", } }, }, diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index e5936c70944f0e..781934d7a6bd9e 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -131,7 +131,8 @@ Now that you have set up the Hubspot source connector, check out the following H ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.2 | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation | | 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | | 0.2.0 | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API. | | 0.1.83 | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated. | From 984945d3aa47d6e2f39d3c6cda197996e49b950d Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Mon, 3 Oct 2022 19:22:03 -0400 Subject: [PATCH 05/32] Added Slack link --- .../config-based/low-code-cdk-overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/connector-development/config-based/low-code-cdk-overview.md b/docs/connector-development/config-based/low-code-cdk-overview.md index 5072045aa504c0..54166a60755895 100644 --- a/docs/connector-development/config-based/low-code-cdk-overview.md +++ b/docs/connector-development/config-based/low-code-cdk-overview.md @@ -3,7 +3,7 @@ Airbyte’s low-code framework enables you to build source connectors for REST APIs by modifying boilerplate YAML files. :::warning -The low-code framework is in [alpha](https://docs.airbyte.com/project-overview/product-release-stages/#alpha), which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests directly with us at [feedback@airbyte.io](mailto:feedback@airbyte.io) +The low-code framework is in [alpha](https://docs.airbyte.com/project-overview/product-release-stages/#alpha), which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests with us on our [Slack channel](https://airbytehq.slack.com/join/shared_invite/zt-1bx4v2t4m-vhZaQu1OdJTyk1M1nSxjBA#/shared-invite/email) or email us at [feedback@airbyte.io](mailto:feedback@airbyte.io) ::: ## What connectors can I build using the low-code framework? @@ -109,4 +109,4 @@ For examples of production-ready config-based connectors, refer to: - [Greenhouse](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-greenhouse) - [Sendgrid](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml) -- [Sentry](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml) \ No newline at end of file +- [Sentry](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml) From e37979b3576d790a3afd12585b5dfdca9e358d1a Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Mon, 3 Oct 2022 19:28:29 -0400 Subject: [PATCH 06/32] Added link to low-code connector framework --- docs/connector-development/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index 4a3773d49f6c6f..c129c1b175b8e4 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -6,6 +6,10 @@ To build a new connector in Java or Python, we provide templates so you don't ne **Note: you are not required to maintain the connectors you create.** The goal is that the Airbyte core team and the community help maintain the connector. +## Low-code Connector-Development Framework + +You can use the [low-code framework](config-based/low-code-cdk-overview.md) to build source connectors for REST APIs by modifying boilerplate YAML files. + ## Python Connector-Development Kit \(CDK\) You can build a connector very quickly in Python with the [Airbyte CDK](cdk-python/), which generates 75% of the code required for you. From 66dafd85aa9c7aa1df02d45974ea8956f5ad2cea Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Mon, 3 Oct 2022 19:39:40 -0400 Subject: [PATCH 07/32] Updated Slack link --- .../connector-development/config-based/low-code-cdk-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/connector-development/config-based/low-code-cdk-overview.md b/docs/connector-development/config-based/low-code-cdk-overview.md index 54166a60755895..6e781a263a221e 100644 --- a/docs/connector-development/config-based/low-code-cdk-overview.md +++ b/docs/connector-development/config-based/low-code-cdk-overview.md @@ -3,7 +3,7 @@ Airbyte’s low-code framework enables you to build source connectors for REST APIs by modifying boilerplate YAML files. :::warning -The low-code framework is in [alpha](https://docs.airbyte.com/project-overview/product-release-stages/#alpha), which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests with us on our [Slack channel](https://airbytehq.slack.com/join/shared_invite/zt-1bx4v2t4m-vhZaQu1OdJTyk1M1nSxjBA#/shared-invite/email) or email us at [feedback@airbyte.io](mailto:feedback@airbyte.io) +The low-code framework is in [alpha](https://docs.airbyte.com/project-overview/product-release-stages/#alpha), which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests with us on our [Slack channel](https://slack.airbyte.com/) or email us at [feedback@airbyte.io](mailto:feedback@airbyte.io) ::: ## What connectors can I build using the low-code framework? From 65cd7e894c5d0e7228b72f2415d6592c7531f7a1 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Mon, 3 Oct 2022 16:46:28 -0700 Subject: [PATCH 08/32] accept null value when evaluating whether to skip normalization (#17537) --- .../temporal/sync/SyncWorkflowImpl.java | 2 +- .../temporal/sync/SyncWorkflowTest.java | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 85016ffa21de35..93b350457eda11 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -69,7 +69,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, Boolean shouldRun; try { shouldRun = normalizationSummaryCheckActivity.shouldRunNormalization(Long.valueOf(jobRunConfig.getJobId()), jobRunConfig.getAttemptId(), - Optional.of(syncOutput.getStandardSyncSummary().getTotalStats().getRecordsCommitted())); + Optional.ofNullable(syncOutput.getStandardSyncSummary().getTotalStats().getRecordsCommitted())); } catch (final IOException e) { shouldRun = true; } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index fe60a4415ae0c8..ada9d6e567d2d4 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -24,6 +24,7 @@ import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; +import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.SyncStats; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; @@ -89,7 +90,9 @@ class SyncWorkflowTest { private NormalizationInput normalizationInput; private OperatorDbtInput operatorDbtInput; private StandardSyncOutput replicationSuccessOutput; + private StandardSyncOutput replicationFailOutput; private StandardSyncSummary standardSyncSummary; + private StandardSyncSummary failedSyncSummary; private SyncStats syncStats; private NormalizationSummary normalizationSummary; private ActivityOptions longActivityOptions; @@ -108,7 +111,10 @@ void setUp() throws IOException { syncStats = new SyncStats().withRecordsCommitted(10L); standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); + failedSyncSummary = new StandardSyncSummary().withStatus(ReplicationStatus.FAILED).withTotalStats(new SyncStats().withRecordsEmitted(0L)); replicationSuccessOutput = new StandardSyncOutput().withOutputCatalog(syncInput.getCatalog()).withStandardSyncSummary(standardSyncSummary); + replicationFailOutput = new StandardSyncOutput().withOutputCatalog(syncInput.getCatalog()).withStandardSyncSummary(failedSyncSummary); + normalizationSummary = new NormalizationSummary(); normalizationInput = new NormalizationInput() @@ -217,6 +223,30 @@ void testReplicationFailure() { verifyNoInteractions(dbtTransformationActivity); } + @Test + void testReplicationFailedGracefully() { + doReturn(replicationFailOutput).when(replicationActivity).replicate( + JOB_RUN_CONFIG, + SOURCE_LAUNCHER_CONFIG, + DESTINATION_LAUNCHER_CONFIG, + syncInput); + + doReturn(normalizationSummary).when(normalizationActivity).normalize( + JOB_RUN_CONFIG, + DESTINATION_LAUNCHER_CONFIG, + normalizationInput); + + final StandardSyncOutput actualOutput = execute(); + + verifyReplication(replicationActivity, syncInput); + verifyPersistState(persistStateActivity, sync, replicationFailOutput, syncInput.getCatalog()); + verifyNormalize(normalizationActivity, normalizationInput); + verifyDbtTransform(dbtTransformationActivity, syncInput.getResourceRequirements(), + operatorDbtInput); + assertEquals(replicationFailOutput.withNormalizationSummary(normalizationSummary), + actualOutput); + } + @Test void testNormalizationFailure() { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( From f45275e78f5ec7dceb05fae0d0d02fffc22dcc10 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 3 Oct 2022 16:58:09 -0700 Subject: [PATCH 09/32] Remove data field in the state aggregator if possible (#17538) * Remove data field in the state aggregator if possible * Add comments * Add comment and update test --- .../SingleStateAggregator.java | 7 ++++ .../StreamStateAggregator.java | 7 ++++ .../state_aggregator/StateAggregatorTest.java | 33 ++++++++++++------- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java index 09106a08c1e243..0cfe422ea1f789 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java @@ -24,6 +24,13 @@ public State getAggregated() { if (state.getType() == null || state.getType() == AirbyteStateType.LEGACY) { return new State().withState(state.getData()); } else { + /** + * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are + * running this code, we know that the platform has been upgraded and we can thus discard the legacy + * state. Keeping the legacy state is causing issue because of its size + * (https://github.com/airbytehq/oncall/issues/731) + */ + state.setData(null); return new State() .withState(Jsons.jsonNode(List.of(state))); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java index d55563efe0ec61..4d3247b2549d01 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java @@ -17,6 +17,13 @@ class StreamStateAggregator implements StateAggregator { @Override public void ingest(final AirbyteStateMessage stateMessage) { + /** + * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are + * running this code, we know that the platform has been upgraded and we can thus discard the legacy + * state. Keeping the legacy state is causing issue because of its size + * (https://github.com/airbytehq/oncall/issues/731) + */ + stateMessage.setData(null); aggregatedState.put(stateMessage.getStream().getStreamDescriptor(), stateMessage); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java index bd591be29b132b..83761bf5b194fc 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java @@ -92,13 +92,16 @@ void testGlobalState() { final AirbyteStateMessage state1 = getGlobalMessage(1); final AirbyteStateMessage state2 = getGlobalMessage(2); - stateAggregator.ingest(state1); + final AirbyteStateMessage state1NoData = getGlobalMessage(1).withData(null); + final AirbyteStateMessage state2NoData = getGlobalMessage(2).withData(null); + + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1)))); + .withState(Jsons.jsonNode(List.of(state1NoData)))); - stateAggregator.ingest(state2); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2)))); + .withState(Jsons.jsonNode(List.of(state2NoData)))); } @Test @@ -126,19 +129,23 @@ void testStreamStateWithFeatureFlagOn() { final AirbyteStateMessage state2 = getStreamMessage("b", 2); final AirbyteStateMessage state3 = getStreamMessage("b", 3); + final AirbyteStateMessage state1NoData = getStreamMessage("a", 1).withData(null); + final AirbyteStateMessage state2NoData = getStreamMessage("b", 2).withData(null); + final AirbyteStateMessage state3NoData = getStreamMessage("b", 3).withData(null); + stateAggregator = new DefaultStateAggregator(USE_STREAM_CAPABLE_STATE); - stateAggregator.ingest(state1); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1)))); + .withState(Jsons.jsonNode(List.of(state1NoData)))); - stateAggregator.ingest(state2); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2, state1)))); + .withState(Jsons.jsonNode(List.of(state2NoData, state1NoData)))); - stateAggregator.ingest(state3); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state3), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state3, state1)))); + .withState(Jsons.jsonNode(List.of(state3NoData, state1NoData)))); } private AirbyteStateMessage getNullMessage(final int stateValue) { @@ -158,7 +165,8 @@ private AirbyteStateMessage getGlobalMessage(final int stateValue) { .withStreamDescriptor( new StreamDescriptor() .withName("test")) - .withStreamState(Jsons.jsonNode(stateValue))))); + .withStreamState(Jsons.jsonNode(stateValue))))) + .withData(Jsons.jsonNode("HelloWorld")); } private AirbyteStateMessage getStreamMessage(final String streamName, final int stateValue) { @@ -168,7 +176,8 @@ private AirbyteStateMessage getStreamMessage(final String streamName, final int .withStreamDescriptor( new StreamDescriptor() .withName(streamName)) - .withStreamState(Jsons.jsonNode(stateValue))); + .withStreamState(Jsons.jsonNode(stateValue))) + .withData(Jsons.jsonNode("Hello")); } private AirbyteStateMessage getEmptyMessage(final AirbyteStateType stateType) { From 61a5a1e9677f1b92557d8540ba6a8cfc77f227f7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 3 Oct 2022 17:02:41 -0700 Subject: [PATCH 10/32] [#17107] Fix propagation of $options field in arrays (#17196) * input can only be a string * remove debug print * cleanup tests * fix in the factory * Add a unit test * fix for lists * Update changelog * Update changelog --- airbyte-cdk/python/CHANGELOG.md | 4 ++ .../sources/declarative/parsers/factory.py | 4 +- airbyte-cdk/python/setup.py | 2 +- .../sources/declarative/test_factory.py | 60 +++++++++++++++++-- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index ef00c23bd5c9b5..af039710854bb3 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.1.92 +- Low-code: Properly propagate $options to array items +- Low-code: Log request and response when running check operation in debug mode + ## 0.1.91 - Low-code: Rename LimitPaginator to DefaultPaginator and move page_size field to PaginationStrategy diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index d31a7d5896b82d..676074d174b3ea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -138,13 +138,11 @@ def build(self, class_or_class_name: Union[str, Type], config, instantiate: bool class_ = self._get_class_from_fully_qualified_class_name(class_or_class_name) else: class_ = class_or_class_name - # create components in options before propagating them if OPTIONS_STR in kwargs: kwargs[OPTIONS_STR] = { k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs[OPTIONS_STR].items() } - updated_kwargs = {k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs.items()} if instantiate: @@ -216,7 +214,7 @@ def _create_subcomponent(self, key, definition, kwargs, config, parent_class, in self._create_subcomponent( key, sub, - self._merge_dicts(kwargs.get(OPTIONS_STR, dict()), self._get_subcomponent_options(sub)), + kwargs, config, parent_class, instantiate, diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 2b833673c5a578..e0c3c6ab9ed5a2 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.91", + version="0.1.92", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index e103aaaea85b20..7cf63571b5ad74 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -328,7 +328,7 @@ def test_full_config(): primary_key: "id" extractor: $ref: "*ref(extractor)" - field_pointer: ["result"] + field_pointer: ["{{ options['name'] }}"] retriever: $ref: "*ref(retriever)" requester: @@ -365,7 +365,7 @@ def test_full_config(): assert type(stream.retriever.record_selector) == RecordSelector assert type(stream.retriever.record_selector.extractor.decoder) == JsonDecoder - assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"] + assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["lists"] assert type(stream.retriever.record_selector.record_filter) == RecordFilter assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}" assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" @@ -378,18 +378,24 @@ def test_full_config(): assert stream.retriever.requester.path.default == "marketing/lists" -def test_create_record_selector(): - content = """ +@pytest.mark.parametrize( + "test_name, record_selector, expected_runtime_selector", + [("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ options['name'] }}", "lists")], +) +def test_create_record_selector(test_name, record_selector, expected_runtime_selector): + content = f""" extractor: type: DpathExtractor selector: + $options: + name: "lists" class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector record_filter: class_name: airbyte_cdk.sources.declarative.extractors.record_filter.RecordFilter condition: "{{ record['id'] > stream_state['id'] }}" extractor: $ref: "*ref(extractor)" - field_pointer: ["result"] + field_pointer: ["{record_selector}"] """ config = parser.parse(content) @@ -398,10 +404,52 @@ def test_create_record_selector(): selector = factory.create_component(config["selector"], input_config)() assert isinstance(selector, RecordSelector) assert isinstance(selector.extractor, DpathExtractor) - assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == ["result"] + assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == [expected_runtime_selector] assert isinstance(selector.record_filter, RecordFilter) +@pytest.mark.parametrize( + "test_name, content, expected_field_pointer_value", + [ + ( + "test_option_in_selector", + """ + extractor: + type: DpathExtractor + field_pointer: ["{{ options['name'] }}"] + selector: + class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector + $options: + name: "selector" + extractor: "*ref(extractor)" + """, + "selector", + ), + ( + "test_option_in_extractor", + """ + extractor: + type: DpathExtractor + $options: + name: "extractor" + field_pointer: ["{{ options['name'] }}"] + selector: + class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector + $options: + name: "selector" + extractor: "*ref(extractor)" + """, + "extractor", + ), + ], +) +def test_options_propagation(test_name, content, expected_field_pointer_value): + config = parser.parse(content) + + selector = factory.create_component(config["selector"], input_config, True)() + assert selector.extractor.field_pointer[0].eval(input_config) == expected_field_pointer_value + + def test_create_requester(): content = """ requester: From 7a7091e2dc6264e393a181b16c474b7633d9782e Mon Sep 17 00:00:00 2001 From: Ryan Fu Date: Mon, 3 Oct 2022 17:54:52 -0700 Subject: [PATCH 11/32] Updated s3.md changelog to include changes for enforcing TLS (#17545) --- docs/integrations/destinations/s3.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index bf6105ff1919e4..882cf1929561d2 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -323,9 +323,10 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.3.15 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields). | +| 0.3.16 | 2022-10-03 | [\#17340](https://github.com/airbytehq/airbyte/pull/17340) | Enforced encrypted only traffic to S3 buckets and check logic | +| 0.3.15 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields). | | 0.3.14 | 2022-08-24 | [\#15207](https://github.com/airbytehq/airbyte/pull/15207) | Fix S3 bucket path to be used for check. | -| 0.3.13 | 2022-08-09 | [\#15394](https://github.com/airbytehq/airbyte/pull/15394) | Added LZO compression support to Parquet format | +| 0.3.13 | 2022-08-09 | [\#15394](https://github.com/airbytehq/airbyte/pull/15394) | Added LZO compression support to Parquet format | | 0.3.12 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.3.11 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.3.10 | 2022-06-30 | [\#14332](https://github.com/airbytehq/airbyte/pull/14332) | Change INSTANCE_PROFILE to use `AWSDefaultProfileCredential`, which supports more authentications on AWS | From 6e235580a5f412f29c69090d7d6c2ba130297f52 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Mon, 3 Oct 2022 18:10:11 -0700 Subject: [PATCH 12/32] Check Replication Summary length (#17546) --- .../workers/temporal/sync/ReplicationActivityImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index 1109fe2487c76e..945e4a63cd694a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -61,6 +61,7 @@ public class ReplicationActivityImpl implements ReplicationActivity { private static final Logger LOGGER = LoggerFactory.getLogger(ReplicationActivityImpl.class); + private static final int MAX_TEMPORAL_MESSAGE_SIZE = 2 * 1024 * 1024; private final Optional containerOrchestratorConfig; private final WorkerConfigs workerConfigs; @@ -156,7 +157,14 @@ public StandardSyncOutput replicate(final JobRunConfig jobRunConfig, final ReplicationOutput attemptOutput = temporalAttempt.get(); final StandardSyncOutput standardSyncOutput = reduceReplicationOutput(attemptOutput); - LOGGER.info("sync summary: {}", standardSyncOutput); + final String standardSyncOutputString = standardSyncOutput.toString(); + LOGGER.info("sync summary: {}", standardSyncOutputString); + if (standardSyncOutputString.length() > MAX_TEMPORAL_MESSAGE_SIZE) { + LOGGER.error("Sync ouput exceeds the max temporal message size of {}, actual is {}.", MAX_TEMPORAL_MESSAGE_SIZE, + standardSyncOutputString.length()); + } else { + LOGGER.info("Sync summary length: {}", standardSyncOutputString.length()); + } return standardSyncOutput; }, From d8b05637230992a12e51c91466ae18281fca1cca Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Mon, 3 Oct 2022 19:02:38 -0700 Subject: [PATCH 13/32] Match entry in client cert keystore to hardcoded postgres driver value (#17515) * Match entry in client cert keystore to hardcoded postgres driver value * Match entry in client cert keystore to hardcoded postgres driver value * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- .../src/main/java/io/airbyte/db/util/SSLCertificateUtils.java | 3 ++- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- docs/integrations/sources/postgres.md | 1 + 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 26a31ff513a5ac..cdc38ccba53060 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -836,7 +836,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.13 + dockerImageTag: 1.0.14 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d7a77544239750..696b7dad236196 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8590,7 +8590,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.13" +- dockerImage: "airbyte/source-postgres:1.0.14" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java index 88eb2c4c5a2d52..29a87ccc52c5ff 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java @@ -41,7 +41,8 @@ public class SSLCertificateUtils { private static final String PKCS_12 = "PKCS12"; private static final String X509 = "X.509"; private static final Random RANDOM = new SecureRandom(); - public static final String KEYSTORE_ENTRY_PREFIX = "ab_"; + // #17000: postgres driver is hardcoded to only load an entry alias "user" + public static final String KEYSTORE_ENTRY_PREFIX = "user"; public static final String KEYSTORE_FILE_NAME = KEYSTORE_ENTRY_PREFIX + "keystore_"; public static final String KEYSTORE_FILE_TYPE = ".p12"; diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 67a8ea2ecb1861..4789c139f5023a 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.13 +LABEL io.airbyte.version=1.0.14 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 64e10cd5aab9cd..6918bbe9b97272 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.13 +LABEL io.airbyte.version=1.0.14 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 9d5d94068daac2..908b3f5619ac18 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -386,6 +386,7 @@ Possible solutions include: | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.14 | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate | | 1.0.13 | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2 | | 1.0.12 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source | | 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor | From c2507303b6dc4945b675d8fb579d42b536dfb8ae Mon Sep 17 00:00:00 2001 From: andriikorotkov <88329385+andriikorotkov@users.noreply.github.com> Date: Tue, 4 Oct 2022 11:40:14 +0300 Subject: [PATCH 14/32] add github action for help-full command processing (#17502) --- .github/workflows/help-full-command.yml | 60 +++++++++++++++++++++++++ .github/workflows/slash-commands.yml | 2 + 2 files changed, 62 insertions(+) create mode 100644 .github/workflows/help-full-command.yml diff --git a/.github/workflows/help-full-command.yml b/.github/workflows/help-full-command.yml new file mode 100644 index 00000000000000..02c38ba6030abc --- /dev/null +++ b/.github/workflows/help-full-command.yml @@ -0,0 +1,60 @@ +name: Run help-full command for e2e-testing-tool +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: true + comment-body: + description: "The comment-body of the slash command. Used to run e2e-testing-tool for given command." + required: true + +jobs: + help-full-run: + runs-on: ubuntu-latest + steps: + - name: Update comment for processing + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + reactions: eyes, rocket + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Pull Testing Tool docker image + run: docker pull airbyte/airbyte-e2e-testing-tool:latest + - name: Create input and output folders + run: | + mkdir secrets + mkdir result + - name: Run docker container with params + run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.inputs.comment-body }} + - name: Read file with results + id: read_file + uses: andstor/file-reader-action@v1 + with: + path: "result/log" + - name: Add Success Comment + if: needs.set-params.outputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + ${{ steps.read_file.outputs.contents }} + reactions: +1 + - name: Add Failure Comment + if: needs.set-params.outputs.comment-id && failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + reactions: -1 \ No newline at end of file diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index e87c6ed15ec750..4a6529537efb64 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -41,10 +41,12 @@ jobs: publish-cdk gke-kube-test run-specific-test + help-full static-args: | repo=${{ steps.getref.outputs.repo }} gitref=${{ steps.getref.outputs.ref }} comment-id=${{ github.event.comment.id }} + comment-body=${{ github.event.comment.body }} dispatch-type: workflow - name: Edit comment with error message From 4a6bc222805f7dd4cb80c461c8713c2d6a064c86 Mon Sep 17 00:00:00 2001 From: andriikorotkov <88329385+andriikorotkov@users.noreply.github.com> Date: Tue, 4 Oct 2022 13:30:49 +0300 Subject: [PATCH 15/32] Update testing-tool help commands (#17553) * add github action for help-full command processing * test pull request for testing testing-tool commands * test pull request for testing testing-tool commands --- .../workflows/commands-for-testing-tool.yml | 87 +++++++++++++++++++ .github/workflows/help-full-command.yml | 60 ------------- .github/workflows/slash-commands.yml | 2 - 3 files changed, 87 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/commands-for-testing-tool.yml delete mode 100644 .github/workflows/help-full-command.yml diff --git a/.github/workflows/commands-for-testing-tool.yml b/.github/workflows/commands-for-testing-tool.yml new file mode 100644 index 00000000000000..481d6ed6e461ee --- /dev/null +++ b/.github/workflows/commands-for-testing-tool.yml @@ -0,0 +1,87 @@ +name: Run Testing Tool Commands + +on: + issue_comment: + types: [created] + +jobs: + set-params: + # Only allow slash commands on pull request (not on issues) + if: ${{ github.event.issue.pull_request }} + runs-on: ubuntu-latest + outputs: + repo: ${{ steps.getref.outputs.repo }} + ref: ${{ steps.getref.outputs.ref }} + comment-id: ${{ steps.comment-info.outputs.comment-id }} + command: ${{ steps.regex.outputs.first_match }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + - name: Get PR repo and ref + id: getref + run: | + pr_info="$(curl ${{ github.event.issue.pull_request.url }})" + echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')" + echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')" + - name: Get comment id + id: comment-info + run: | + echo ::set-output name=comment-id::"${{ github.event.comment.id }}" + - name: Get command + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^/[a-zA-Z0-9_/-]+" + regex_flags: "i" + search_string: ${{ github.event.comment.body }} + helps-run: + runs-on: ubuntu-latest + if: | + needs.set-params.outputs.command == '/help-full' || + needs.set-params.outputs.command == '/help' + needs: set-params + steps: + - name: Update comment for processing + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ needs.set-params.outputs.comment-id }} + reactions: eyes, rocket + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Pull Testing Tool docker image + run: docker pull airbyte/airbyte-e2e-testing-tool:latest + - name: Create input and output folders + run: | + mkdir secrets + mkdir result + - name: Run docker container with params + run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }} + - name: Read file with results + id: read_file + uses: andstor/file-reader-action@v1 + with: + path: "result/log" + - name: Add Success Comment + if: needs.set-params.outputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ needs.set-params.outputs.comment-id }} + body: | + > :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + ${{ steps.read_file.outputs.contents }} + reactions: +1 + - name: Add Failure Comment + if: needs.set-params.outputs.comment-id && failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ needs.set-params.outputs.comment-id }} + body: | + > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + reactions: -1 \ No newline at end of file diff --git a/.github/workflows/help-full-command.yml b/.github/workflows/help-full-command.yml deleted file mode 100644 index 02c38ba6030abc..00000000000000 --- a/.github/workflows/help-full-command.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Run help-full command for e2e-testing-tool -on: - workflow_dispatch: - inputs: - repo: - description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." - required: false - default: "airbytehq/airbyte" - gitref: - description: "The git ref to check out from the specified repository." - required: false - default: master - comment-id: - description: "The comment-id of the slash command. Used to update the comment with the status." - required: true - comment-body: - description: "The comment-body of the slash command. Used to run e2e-testing-tool for given command." - required: true - -jobs: - help-full-run: - runs-on: ubuntu-latest - steps: - - name: Update comment for processing - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - reactions: eyes, rocket - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Pull Testing Tool docker image - run: docker pull airbyte/airbyte-e2e-testing-tool:latest - - name: Create input and output folders - run: | - mkdir secrets - mkdir result - - name: Run docker container with params - run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.inputs.comment-body }} - - name: Read file with results - id: read_file - uses: andstor/file-reader-action@v1 - with: - path: "result/log" - - name: Add Success Comment - if: needs.set-params.outputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - ${{ steps.read_file.outputs.contents }} - reactions: +1 - - name: Add Failure Comment - if: needs.set-params.outputs.comment-id && failure() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - reactions: -1 \ No newline at end of file diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 4a6529537efb64..e87c6ed15ec750 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -41,12 +41,10 @@ jobs: publish-cdk gke-kube-test run-specific-test - help-full static-args: | repo=${{ steps.getref.outputs.repo }} gitref=${{ steps.getref.outputs.ref }} comment-id=${{ github.event.comment.id }} - comment-body=${{ github.event.comment.body }} dispatch-type: workflow - name: Edit comment with error message From 7126f8ab14043112e9f9f300b185f3dbcf4d5fa0 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Tue, 4 Oct 2022 14:16:37 +0200 Subject: [PATCH 16/32] Source Iterable: Docs update (prerequisites and setup) (#17533) --- docs/integrations/sources/iterable.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index aee1db7dae4292..ff08803e187d57 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -5,12 +5,13 @@ This page contains the setup guide and reference information for the Iterable so ## Prerequisites * Iterable Account -* Iterable API Key with `standard` permissions. See [API Keys docs](https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-) for more details. +* Iterable `Server-side` API Key with `standard` permissions. See [API Keys docs](https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-) for more details. ## Setup guide ### Step 1: Set up Iterable -Please read [How to find your API key](https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-#creating-api-keys). +* Please read [How to find your API key](https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-#creating-api-keys). +* Make sure that selected API Key has sufficient **permissions** to read all selected [streams](https://api.iterable.com/api/docs#). ## Step 2: Set up the Iterable connector in Airbyte ### For Airbyte Cloud: From 87bba8e1fb8c9482480bf5fe456c23472a4aa65f Mon Sep 17 00:00:00 2001 From: Conor Date: Tue, 4 Oct 2022 10:05:06 -0500 Subject: [PATCH 17/32] ci: build and publish cron with other oss images (#17560) This is needed for cron to be consumed downstream with the correct tag in cloud --- docker-compose-cloud.buildx.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker-compose-cloud.buildx.yaml b/docker-compose-cloud.buildx.yaml index a8c91c97a66e3d..a971427ea7cca5 100644 --- a/docker-compose-cloud.buildx.yaml +++ b/docker-compose-cloud.buildx.yaml @@ -84,3 +84,19 @@ services: platforms: - linux/amd64 - linux/arm64 + cron: + image: airbyte/cron:${VERSION} + build: + dockerfile: Dockerfile + context: airbyte-cron/build/docker + labels: + io.airbyte.git-revision: ${GIT_REVISION} + args: + VERSION: ${VERSION} + x-bake: + tags: + - airbyte/cron:${VERSION} + - airbyte/cron:${ALT_TAG:-${VERSION}} + platforms: + - linux/amd64 + - linux/arm64 From 5568dd60adde3713119cafb33293cfd253f4a4b4 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Tue, 4 Oct 2022 20:01:49 +0400 Subject: [PATCH 18/32] Source paypal transaction typeerror (#17554) * Source Paypal Transaction: fix source config accordion to specification * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-paypal-transaction/Dockerfile | 2 +- .../acceptance-test-docker.sh | 2 +- .../source_paypal_transaction/source.py | 5 ++-- .../unit_tests/conftest.py | 4 +-- .../sources/paypal-transaction.md | 25 ++++++++++--------- 7 files changed, 22 insertions(+), 20 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index cdc38ccba53060..ce8240ff95c467 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -765,7 +765,7 @@ - name: Paypal Transaction sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 dockerRepository: airbyte/source-paypal-transaction - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction icon: paypal.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 696b7dad236196..0951e5a663e0c1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8179,7 +8179,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-paypal-transaction:0.1.9" +- dockerImage: "airbyte/source-paypal-transaction:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/paypal-transactions" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile index a791e3f3d2209b..9e26b7a545bcf3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh old mode 100644 new mode 100755 index e4d8b1cef8961e..c51577d10690c1 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index e3b7c09d9057c8..9606eb2de99df4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -442,10 +442,11 @@ def __init__(self, config: Dict): "refresh_token": "", } # support old configs - if "client_id" and "secret" in config: + if "client_id" in config and ("client_secret" in config or "secret" in config): self.old_config = True + client_secret = config.get("client_secret", config.get("secret")) self.auth_args.update( - **{"client_id": config["client_id"], "client_secret": config["secret"], "refresh_token": config.get("refresh_token")} + **{"client_id": config["client_id"], "client_secret": client_secret, "refresh_token": config.get("refresh_token")} ) # new configs if "credentials" in config: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index 3d3f6692055113..8bd9555a4029d0 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -40,7 +40,7 @@ def prod_config(): """ return { "client_id": "some_client_id", - "secret": "some_secret", + "client_secret": "some_secret", "start_date": "2021-07-01T00:00:00+00:00", "end_date": "2021-07-10T00:00:00+00:00", "is_sandbox": False, @@ -54,7 +54,7 @@ def sandbox_config(): """ return { "client_id": "some_client_id", - "secret": "some_secret", + "client_secret": "some_secret", "start_date": "2021-07-01T00:00:00+00:00", "end_date": "2021-07-10T00:00:00+00:00", "is_sandbox": True, diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index d72e5dc168b715..46297dc290f6cc 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -79,15 +79,16 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------| -| 0.1.9 | 2022-08-18 | [15741](https://github.com/airbytehq/airbyte/pull/15741) | Removed `OAuth2.0` option | -| 0.1.8 | 2022-07-25 | [15000](https://github.com/airbytehq/airbyte/pull/15000) | Added support of `OAuth2.0` authentication, fixed bug when normalization couldn't handle nested cursor field and primary key | -| 0.1.7 | 2022-07-18 | [14804](https://github.com/airbytehq/airbyte/pull/14804) | Added `RESULTSET_TOO_LARGE` error validation | -| 0.1.6 | 2022-06-10 | [13682](https://github.com/airbytehq/airbyte/pull/13682) | Updated paypal transaction schema | -| 0.1.5 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Added fixtures to mock time.sleep for connectors that explicitly sleep | -| 0.1.4 | 2021-12-22 | [9034](https://github.com/airbytehq/airbyte/pull/9034) | Updated connector fields title/description | -| 0.1.3 | 2021-12-16 | [8580](https://github.com/airbytehq/airbyte/pull/8580) | Added more logs during `check connection` stage | -| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Removed base-python dependencies | -| 0.1.1 | 2021-08-03 | [5155](https://github.com/airbytehq/airbyte/pull/5155) | Fixed start_date_min limit | -| 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------| +| 0.1.10 | 2022-09-04 | [17554](https://github.com/airbytehq/airbyte/pull/17554) | Made the spec and source config to be consistent | +| 0.1.9 | 2022-08-18 | [15741](https://github.com/airbytehq/airbyte/pull/15741) | Removed `OAuth2.0` option | +| 0.1.8 | 2022-07-25 | [15000](https://github.com/airbytehq/airbyte/pull/15000) | Added support of `OAuth2.0` authentication, fixed bug when normalization couldn't handle nested cursor field and primary key | +| 0.1.7 | 2022-07-18 | [14804](https://github.com/airbytehq/airbyte/pull/14804) | Added `RESULTSET_TOO_LARGE` error validation | +| 0.1.6 | 2022-06-10 | [13682](https://github.com/airbytehq/airbyte/pull/13682) | Updated paypal transaction schema | +| 0.1.5 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Added fixtures to mock time.sleep for connectors that explicitly sleep | +| 0.1.4 | 2021-12-22 | [9034](https://github.com/airbytehq/airbyte/pull/9034) | Updated connector fields title/description | +| 0.1.3 | 2021-12-16 | [8580](https://github.com/airbytehq/airbyte/pull/8580) | Added more logs during `check connection` stage | +| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Removed base-python dependencies | +| 0.1.1 | 2021-08-03 | [5155](https://github.com/airbytehq/airbyte/pull/5155) | Fixed start_date_min limit | +| 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | From 1d3eadc91a64c31f8a17b90228692caba97a37de Mon Sep 17 00:00:00 2001 From: Arsen Losenko <20901439+arsenlosenko@users.noreply.github.com> Date: Tue, 4 Oct 2022 19:09:27 +0300 Subject: [PATCH 19/32] Source Paypal Transaction: change releaseStage to GA (#17046) --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index ce8240ff95c467..3389b44098aad5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -769,7 +769,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction icon: paypal.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Paystack sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b dockerRepository: airbyte/source-paystack From 4ccbc67ffff4e381ea801433a35c7f34c71ce880 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Tue, 4 Oct 2022 13:22:23 -0400 Subject: [PATCH 20/32] Use constant for keystore prefix in test (#17571) --- .../test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java index f77e8452f886dc..0af7ec33251b9f 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java @@ -171,7 +171,7 @@ void testKeyStoreFromClientCertificateInternal( final KeyStore ks = KeyStore.getInstance("PKCS12"); final InputStream inputStream = Files.newInputStream(Path.of(ksUri)); ks.load(inputStream, KEY_STORE_PASSWORD.toCharArray()); - assertTrue(ks.isKeyEntry("ab_")); + assertTrue(ks.isKeyEntry(SSLCertificateUtils.KEYSTORE_ENTRY_PREFIX)); assertFalse(ks.isKeyEntry("cd_")); assertEquals(1, ks.size()); Files.delete(Path.of(ksUri)); From f8987a0f2c20e91643ff3edfa57c1fb797dac2e3 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Tue, 4 Oct 2022 20:29:39 +0300 Subject: [PATCH 21/32] Source Facebook Marketing: Add `cursor_field` for custom_insights stream schema (#17551) Signed-off-by: Sergey Chvalyuk --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-facebook-marketing/Dockerfile | 2 +- .../source_facebook_marketing/streams/base_insight_streams.py | 2 +- .../unit_tests/test_base_insight_streams.py | 2 ++ docs/integrations/sources/facebook-marketing.md | 1 + 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 3389b44098aad5..1a3d4d04de2d3d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -279,7 +279,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.66 + dockerImageTag: 0.2.67 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 0951e5a663e0c1..968a96fbef10d9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2561,7 +2561,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.66" +- dockerImage: "airbyte/source-facebook-marketing:0.2.67" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 2c0323f5826eb5..c60b9cffd81d9a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.66 +LABEL io.airbyte.version=0.2.67 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 3754a699a8ffee..1c074a33be4f97 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -265,7 +265,7 @@ def get_json_schema(self) -> Mapping[str, Any]: loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) schema = loader.get_schema("ads_insights") if self._fields: - schema["properties"] = {k: v for k, v in schema["properties"].items() if k in self._fields} + schema["properties"] = {k: v for k, v in schema["properties"].items() if k in self._fields + [self.cursor_field]} if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 2b0e8cbd2e5873..6cc9259e66a98a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -290,3 +290,5 @@ def test_fields_custom(self, api): ) assert stream.fields == ["account_id", "account_currency"] + schema = stream.get_json_schema() + assert schema["properties"].keys() == set(["account_currency", "account_id", stream.cursor_field]) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 9862820c4a2847..7bccc771b1aeca 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -121,6 +121,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 0.2.67 | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema | | 0.2.65 | 2022-09-29 | [17371](https://github.com/airbytehq/airbyte/pull/17371) | Fix stream CustomConversions `enable_deleted=False` | | 0.2.64 | 2022-09-22 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | 0.2.64 | 2022-09-22 | [17027](https://github.com/airbytehq/airbyte/pull/17027) | Limit time range with 37 months when creating an insight job from lower edge object. Retry bulk request when getting error code `960` | From 60628608f14764367b22de083953639034a062a8 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 4 Oct 2022 21:07:13 +0300 Subject: [PATCH 22/32] Source Slack, Source Tiktok-marketing - fix builds (#17557) * #17506 Source Slack, Source Tiktok-marketing - fix builds * source tiktok marketing: upd changelog * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-slack/acceptance-test-config.yml | 2 - .../integration_tests/expected_records.txt | 35 ------------ .../source-tiktok-marketing/Dockerfile | 2 +- .../integration_tests/streams_basic.json | 18 ++++-- .../streams_reports_daily.json | 56 +++++++++++++------ .../streams_reports_lifetime.json | 35 ++++++++---- .../streams_with_day_granularity.json | 56 +++++++++++++------ .../streams_with_lifetime_granularity.json | 35 ++++++++---- .../source_tiktok_marketing/streams.py | 3 +- .../unit_tests/unit_test.py | 5 +- docs/integrations/sources/tiktok-marketing.md | 3 +- 13 files changed, 152 insertions(+), 102 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1a3d4d04de2d3d..5cd8b014c1b2d9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1059,7 +1059,7 @@ - name: TikTok Marketing sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 dockerRepository: airbyte/source-tiktok-marketing - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing icon: tiktok.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 968a96fbef10d9..734add6ec0dcb8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11181,7 +11181,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tiktok-marketing:0.1.16" +- dockerImage: "airbyte/source-tiktok-marketing:0.1.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index a222ae64692dcd..76cc642d536de1 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -18,8 +18,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_catalog.json" - expect_records: - path: "integration_tests/expected_records.txt" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_catalog.json" diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt deleted file mode 100644 index 23d1bbd4a23403..00000000000000 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt +++ /dev/null @@ -1,35 +0,0 @@ -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T01AB4DDR2N", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397747} -{"stream": "users", "data": {"id": "U019WF8P78X", "team_id": "T01AB4DDR2N", "name": "google_calendar", "deleted": false, "color": "3c989f", "real_name": "Google Calendar", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Google Calendar", "real_name_normalized": "Google Calendar", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "27bf72bdda29", "api_app_id": "ADZ494LHY", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_original.png", "is_custom_image": true, "bot_id": "B01APS4667K", "first_name": "Google", "last_name": "Calendar", "image_24": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_24.png", "image_32": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_32.png", "image_48": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_48.png", "image_72": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_72.png", "image_192": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_192.png", "image_512": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_512.png", "image_1024": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1622570515, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397751} -{"stream": "users", "data": {"id": "U019WFB2G31", "team_id": "T01AB4DDR2N", "name": "simplepoll", "deleted": false, "color": "674b1b", "real_name": "Simple Poll", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Simple Poll", "real_name_normalized": "Simple Poll", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "0eee8341dcfc", "api_app_id": "A0HFW7MR6", "always_active": true, "image_original": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_original.png", "is_custom_image": true, "bot_id": "B01ABDTVBJ7", "first_name": "Simple", "last_name": "Poll", "image_24": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_24.png", "image_32": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_32.png", "image_48": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_48.png", "image_72": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_72.png", "image_192": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_192.png", "image_512": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_512.png", "image_1024": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1661533967, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397752} -{"stream": "users", "data": {"id": "U01A6TEQXRU", "team_id": "T01AB4DDR2N", "name": "sherif", "deleted": false, "color": "5b89d5", "real_name": "sherif (please use other account)", "tz": "Asia/Beirut", "tz_label": "Eastern European Summer Time", "tz_offset": 10800, "profile": {"title": "Connectors Engineering", "phone": "", "skype": "", "real_name": "sherif (please use other account)", "real_name_normalized": "sherif (please use other account)", "display_name": "s", "display_name_normalized": "s", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "6613ce572ac2", "image_original": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_original.jpg", "is_custom_image": true, "email": "sherif@airbyte.io", "huddle_state": "default_unset", "first_name": "sherif", "last_name": "(please use other account)", "image_24": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664337135, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397753} -{"stream": "users", "data": {"id": "U01A81VHD45", "team_id": "T01AB4DDR2N", "name": "michel", "deleted": false, "color": "9f69e7", "real_name": "Michel Tricot", "tz": "Europe/Brussels", "tz_label": "Central European Summer Time", "tz_offset": 7200, "profile": {"title": "Co-Founder & CEO", "phone": "", "skype": "", "real_name": "Michel Tricot", "real_name_normalized": "Michel Tricot", "display_name": "Michel", "display_name_normalized": "Michel", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "9cec7ecdfa05", "image_original": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_original.jpg", "is_custom_image": true, "email": "michel@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "Michel", "last_name": "Tricot", "image_24": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": true, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664190286, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397754} -{"stream": "users", "data": {"id": "U01AB6V6NMQ", "team_id": "T01AB4DDR2N", "name": "john", "deleted": false, "color": "4bbe2e", "real_name": "John Lafleur", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Co-founder", "phone": "4153597503", "skype": "", "real_name": "John Lafleur", "real_name_normalized": "John Lafleur", "display_name": "John (Airbyte)", "display_name_normalized": "John (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "6a64a9108cd2", "image_original": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_original.jpg", "is_custom_image": true, "email": "john@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "John", "last_name": "Lafleur", "image_24": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1663767942, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397755} -{"stream": "users", "data": {"id": "U01ABDY04RZ", "team_id": "T01AB4DDR2N", "name": "github-x", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "GitHub (Legacy)", "real_name_normalized": "GitHub (Legacy)", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "de0095e0c631", "api_app_id": "A8GBNUWU8", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_original.png", "is_custom_image": true, "bot_id": "B01A85CCXT7", "first_name": "GitHub", "last_name": "(Legacy)", "image_24": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_bot": true, "is_app_user": true, "updated": 1618854795}, "emitted_at": 1661589397756} -{"stream": "users", "data": {"id": "U01ADKHBJKC", "team_id": "T01AB4DDR2N", "name": "jared", "deleted": false, "color": "e0a729", "real_name": "Jared (message me on other account)", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Engineering (Platform) @ Airbyte", "phone": "", "skype": "", "real_name": "Jared (message me on other account)", "real_name_normalized": "Jared (message me on other account)", "display_name": "Jared Rhizor (Airbyte)", "display_name_normalized": "Jared Rhizor (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "438b7f2a6e75", "image_original": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_original.jpg", "is_custom_image": true, "email": "jared@airbyte.io", "first_name": "Jared", "last_name": "(message me on other account)", "image_24": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549241, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397757} -{"stream": "users", "data": {"id": "U01ADQ3FGMC", "team_id": "T01AB4DDR2N", "name": "zapier", "deleted": false, "color": "df3dc0", "real_name": "Zapier", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Zapier", "real_name_normalized": "Zapier", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "302ec95148aa", "api_app_id": "A024R9PQM", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_original.png", "is_custom_image": true, "bot_id": "B01ALN9M9GT", "first_name": "Zapier", "last_name": "", "image_24": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1662549246, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397758} -{"stream": "users", "data": {"id": "U01ADSJ1D5H", "team_id": "T01AB4DDR2N", "name": "jamakase54", "deleted": false, "color": "2b6836", "real_name": "Artem Astapenko", "tz": "Europe/Moscow", "tz_label": "Moscow Time", "tz_offset": 10800, "profile": {"title": "CEO and Tech Lead @ Jamakase Technologies", "phone": "", "skype": "", "real_name": "Artem Astapenko", "real_name_normalized": "Artem Astapenko", "display_name": "Artem Astapenko", "display_name_normalized": "Artem Astapenko", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "69713daee7ef", "image_original": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_original.jpg", "is_custom_image": true, "email": "jamakase54@gmail.com", "first_name": "Artem", "last_name": "Astapenko", "image_24": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_24.jpg", "image_32": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_32.jpg", "image_48": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_48.jpg", "image_72": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_72.jpg", "image_192": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_192.jpg", "image_512": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_512.jpg", "image_1024": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549243, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397759} -{"stream": "channel_messages", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589426355} -{"stream": "channel_messages", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "3e57321f-da00-49dc-94b5-f3a0ef5aa723", "type": "message", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?", "user": "U027CJETYP8", "ts": "1627033140.340300", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "k=mGP", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?"}]}]}], "thread_ts": "1627033140.340300", "reply_count": 5, "reply_users_count": 3, "latest_reply": "1627934001.030300", "reply_users": ["U01BV1SDQMA", "U027CJETYP8", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627033140.3403}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "092f4adb-d803-47ae-8c36-f6e2e99141b0", "type": "message", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n```kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\n created\n created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. ```\nAre there any workarounds on this issue?", "user": "U024WGT8H3R", "ts": "1627023042.338300", "team": "T01AB4DDR2N", "edited": {"user": "U024WGT8H3R", "ts": "1627023114.000000"}, "blocks": [{"type": "rich_text", "block_id": "4NDy", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n"}]}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\nrole.rbac.authorization.k8s.io/airbyte-admin-role created\nrolebinding.rbac.authorization.k8s.io/airbyte-admin-binding created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. "}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Are there any workarounds on this issue?"}]}]}], "thread_ts": "1627023042.338300", "reply_count": 9, "reply_users_count": 4, "latest_reply": "1632724785.457800", "reply_users": ["U01BV1SDQMA", "U01HYQYV1BQ", "U024WGT8H3R", "U02F7M6UD0S"], "is_locked": false, "subscribed": false, "reactions": [{"name": "white_check_mark", "users": ["U01HYQYV1BQ"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627023042.3383}, "emitted_at": 1661589426858} -{"stream": "threads", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589428491} -{"stream": "threads", "data": {"client_msg_id": "6ae996d2-85c7-423f-b24e-9f6ce29c109e", "type": "message", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article.", "user": "U024VC21YRW", "ts": "1627055946.177500", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "4xpJB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article."}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "channel_id": "C01AB7G87NE", "float_ts": 1627055946.1775}, "emitted_at": 1661589428494} -{"stream": "threads", "data": {"client_msg_id": "00b54cbd-9fce-421d-9c4e-33a5bad08ab7", "type": "message", "text": "<@U01RW78A2LE> this is something that maybe we need to discuss :smile:", "user": "U01MMSDJGC9", "ts": "1627331109.183100", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "xBQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U01RW78A2LE"}, {"type": "text", "text": " this is something that maybe we need to discuss "}, {"type": "emoji", "name": "smile", "unicode": "1f604"}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "reactions": [{"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627331109.1831}, "emitted_at": 1661589428494} -{"stream": "threads", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589429206} -{"stream": "threads", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589429515} -{"stream": "threads", "data": {"client_msg_id": "71a5a159-5e5c-462d-87ce-c202172dbb99", "type": "message", "text": "You can follow these issues, i think they are similar to what you are reporting:\n\u2022 \n\u2022 ", "user": "U01BV1SDQMA", "ts": "1627060931.349300", "team": "T01AB4DDR2N", "attachments": [{"title": "", "footer": "", "id": 1, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1621556593, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/3520", "is_app_unfurl": true}, {"title": "", "footer": "", "id": 2, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1624460581, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/4295", "is_app_unfurl": true}], "blocks": [{"type": "rich_text", "block_id": "5BgK", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "You can follow these issues, i think they are similar to what you are reporting:\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/3520"}]}, {"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/4295"}]}], "style": "bullet", "indent": 0}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627060931.3493}, "emitted_at": 1661589429516} -{"stream": "threads", "data": {"client_msg_id": "bea4bda9-286a-494d-b9a0-e21799482588", "type": "message", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself", "user": "U01BV1SDQMA", "ts": "1627061101.349600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "a/t", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "reactions": [{"name": "+1", "users": ["U028HUR1TJR"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627061101.3496}, "emitted_at": 1661589429517} -{"stream": "threads", "data": {"client_msg_id": "ba278400-8e9a-4602-9a9e-8d19b4119d5b", "type": "message", "text": "Thanks !", "user": "U028HUR1TJR", "ts": "1627062388.350800", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "vXY6K", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627062388.3508}, "emitted_at": 1661589429518} -{"stream": "threads", "data": {"client_msg_id": "3e57321f-da00-49dc-94b5-f3a0ef5aa723", "type": "message", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?", "user": "U027CJETYP8", "ts": "1627033140.340300", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "k=mGP", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?"}]}]}], "thread_ts": "1627033140.340300", "reply_count": 5, "reply_users_count": 3, "latest_reply": "1627934001.030300", "reply_users": ["U01BV1SDQMA", "U027CJETYP8", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627033140.3403}, "emitted_at": 1661589429820} -{"stream": "threads", "data": {"client_msg_id": "98339578-a93b-42ba-b325-271dbe71bfb4", "type": "message", "text": "have you gone through: ?", "user": "U01BV1SDQMA", "ts": "1627033374.340400", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "F=5uE", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "have you gone through: "}, {"type": "link", "url": "https://docs.airbyte.io/understanding-airbyte/namespaces"}, {"type": "text", "text": " ?"}]}]}], "thread_ts": "1627033140.340300", "parent_user_id": "U027CJETYP8", "channel_id": "C021JANJ6TY", "float_ts": 1627033374.3404}, "emitted_at": 1661589429821} -{"stream": "channel_members", "data": {"member_id": "U01A6TEQXRU", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431665} -{"stream": "channel_members", "data": {"member_id": "U01A81VHD45", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431667} -{"stream": "channel_members", "data": {"member_id": "U01AB6V6NMQ", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431668} -{"stream": "channel_members", "data": {"member_id": "U01ADKHBJKC", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431668} -{"stream": "channel_members", "data": {"member_id": "U01ADSJ1D5H", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431669} -{"stream": "channel_members", "data": {"member_id": "U01AS8LGX41", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431669} -{"stream": "channel_members", "data": {"member_id": "U01AV3Y2MK9", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01AW6ZB6KZ", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01B0DNF207", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01B22CCBF0", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431671} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 67ad709b1674fc..0268dfbb1bd582 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json index 6c4951dc1bc540..1fe24b09642a3c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json @@ -8,9 +8,9 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "primary_key": [["id"]] }, - { "stream": { "name": "ads", @@ -18,10 +18,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["modify_time"], - "source_defined_primary_key": [["ads"]] + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["modify_time"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -33,7 +35,9 @@ "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["modify_time"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -45,7 +49,9 @@ "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "primary_key": [["campaign_id"]], + "cursor_field": ["modify_time"] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json index 1514e91196b3a3..e39a974ba78e56 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { "stream": { @@ -62,10 +77,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -73,10 +91,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -84,10 +105,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json index 6738190a078c03..bbc7be4393521b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json index 47dcd3c6c1f95f..da31538ef919b1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { "stream": { @@ -62,10 +77,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -73,10 +91,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -84,10 +105,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json index f7c98e21909c6a..26a9bc1410674e 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index e10e0eef777958..855658f596f406 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -198,13 +198,14 @@ def should_retry(self, response: requests.Response) -> bool: """ Once the rate limit is met, the server returns "code": 40100 Docs: https://business-api.tiktok.com/marketing_api/docs?id=1701890997610497 + Retry 50002 as well - it's a server error. """ try: data = response.json() except Exception: self.logger.error(f"Incorrect JSON response: {response.text}") raise - if data["code"] == 40100: + if data["code"] in (40100, 50002): return True return super().should_retry(response) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index ef7eb38767d79a..17efde817bf8ef 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -35,13 +35,14 @@ def prepared_prod_args(): @timeout_decorator.timeout(20) -def test_backoff(prepared_sandbox_args): +@pytest.mark.parametrize("error_code", (40100, 50002)) +def test_backoff(prepared_sandbox_args, error_code): """TiktokMarketing sends the header 'Retry-After' about needed delay. All streams have to handle it""" stream = Advertisers(**prepared_sandbox_args) with requests_mock.Mocker() as m: url = stream.url_base + stream.path() - m.get(url, text=json.dumps({"code": 40100})) + m.get(url, text=json.dumps({"code": error_code})) with pytest.raises(UserDefinedBackoffException): list(stream.read_records(sync_mode=None)) diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 3f64914edcfb7a..1eae546492d6f1 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -536,7 +536,8 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | | 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | | 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | | 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | From 8b4f65de9e7b461953e9a80c0dc0d0970f8ada42 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Tue, 4 Oct 2022 21:50:20 +0300 Subject: [PATCH 23/32] Source Github: Skip repository if got HTTP 500 for WorkflowRuns stream (#17555) Signed-off-by: Sergey Chvalyuk --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- airbyte-integrations/connectors/source-github/Dockerfile | 2 +- .../connectors/source-github/source_github/streams.py | 2 ++ docs/integrations/sources/github.md | 1 + 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5cd8b014c1b2d9..d9136a0c2c55aa 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -354,7 +354,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.4 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 734add6ec0dcb8..e048fe4975f30d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3503,7 +3503,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.3.3" +- dockerImage: "airbyte/source-github:0.3.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/github" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 112569d3466096..c8d1ce128be979 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.3 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 7c495444a067f7..22a2136b7bf0cf 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -152,6 +152,8 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter f"Syncing `{self.name}` stream isn't available for repository " f"`{stream_slice['repository']}`, it seems like this repository is empty." ) + elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns): + error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`." else: # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text self.logger.error(f"Undefined error while reading records: {e.response.text}") diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index cddeefb74315ef..df06070a6beec9 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -147,6 +147,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa | Version | Date | Pull Request | Subject | | :------ | :--------- | :---------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 0.3.4 | 2022-10-04 | [17555](https://github.com/airbytehq/airbyte/pull/17555) | Skip repository if got HTTP 500 for WorkflowRuns stream | | 0.3.3 | 2022-09-28 | [17287](https://github.com/airbytehq/airbyte/pull/17287) | Fix problem with "null" `cursor_field` for WorkflowJobs stream | | 0.3.2 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | 0.3.1 | 2022-09-21 | [16947](https://github.com/airbytehq/airbyte/pull/16947) | Improve error logging when handling HTTP 500 error | From aedd535ab7272867241970790c9b8f9f8945bf78 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 4 Oct 2022 12:05:33 -0700 Subject: [PATCH 24/32] Disable normalization skip when no new data (#17574) * Disable normalization skip when no new data * Disable irrelevant tests --- .../io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java | 6 ++++-- .../io/airbyte/workers/temporal/sync/SyncWorkflowTest.java | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 93b350457eda11..5aa79399486737 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -74,8 +74,10 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, shouldRun = true; } if (!shouldRun) { - LOGGER.info("Skipping normalization because there are no records to normalize."); - continue; + LOGGER.info("No records to normalize detected"); + // Normalization skip has been disabled: issue #5417 + // LOGGER.info("Skipping normalization because there are no records to normalize."); + // continue; } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index ada9d6e567d2d4..1e29735bc31677 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -50,6 +50,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @SuppressWarnings({"PMD.UnusedPrivateField", "PMD.UnusedPrivateMethod"}) @@ -312,6 +313,7 @@ void testCancelDuringNormalization() { } @Test + @Disabled("This behavior has been disabled temporarily (OC Issue #741)") void testSkipNormalization() throws IOException { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); From 759a27ec453debbaefa623292995befc6db3c0b9 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Tue, 4 Oct 2022 15:14:13 -0400 Subject: [PATCH 25/32] Use Micronaut auto-detected environment for docker vs kubernetes choices (#17388) * Use Micronaut auto-detected environment for docker vs kubernetes choices * Handle set, but blank env var * Formatting * Revert change * Add default value * Explicitly set WORKER_ENVIRONMENT for container orchestrator * Make public * Add sleep to allow Temporal cache to populate Co-authored-by: Jimmy Ma --- .../temporal/TemporalInitializationUtils.java | 5 +- .../ContainerOrchestratorApp.java | 4 +- .../DbtJobOrchestrator.java | 2 - .../NormalizationJobOrchestrator.java | 6 +-- .../ReplicationJobOrchestrator.java | 8 +--- .../DestinationAcceptanceTest.java | 19 ++++---- .../source/AbstractSourceConnectorTest.java | 10 ++-- .../source/PythonSourceAcceptanceTest.java | 4 +- .../workers/ApplicationInitializer.java | 2 +- .../workers/ContainerOrchestratorConfig.java | 4 +- .../java/io/airbyte/workers/WorkerUtils.java | 7 ++- .../config/ApplicationBeanFactory.java | 5 +- ...ontainerOrchestratorConfigBeanFactory.java | 7 ++- .../config/ProcessFactoryBeanFactory.java | 38 ++++++--------- .../WorkerConfigurationBeanFactory.java | 9 ++-- .../general/DbtTransformationRunner.java | 8 +--- .../general/DefaultCheckConnectionWorker.java | 12 ++--- .../general/DefaultDiscoverCatalogWorker.java | 14 +++--- .../workers/general/DefaultGetSpecWorker.java | 12 ++--- .../internal/DefaultAirbyteDestination.java | 12 ++--- .../internal/DefaultAirbyteSource.java | 11 ++--- .../DefaultNormalizationRunner.java | 8 +--- .../NormalizationRunnerFactory.java | 5 +- .../workers/process/DockerProcessFactory.java | 2 +- .../CheckConnectionActivityImpl.java | 2 +- .../catalog/DiscoverCatalogActivityImpl.java | 2 +- .../temporal/spec/SpecActivityImpl.java | 2 +- .../sync/DbtTransformationActivityImpl.java | 2 - .../workers/temporal/sync/LauncherWorker.java | 4 ++ .../sync/NormalizationActivityImpl.java | 4 +- .../sync/ReplicationActivityImpl.java | 8 +--- .../src/main/resources/application.yml | 3 +- .../DefaultCheckConnectionWorkerTest.java | 16 +++---- .../DefaultDiscoverCatalogWorkerTest.java | 14 ++---- .../general/DefaultGetSpecWorkerTest.java | 4 +- .../DefaultAirbyteDestinationTest.java | 14 ++---- .../internal/DefaultAirbyteSourceTest.java | 12 ++--- .../DefaultNormalizationRunnerTest.java | 46 +++++++++---------- .../NormalizationRunnerFactoryTest.java | 5 +- .../process/DockerProcessFactoryTest.java | 4 +- .../airbyte-worker/templates/deployment.yaml | 5 -- docker-compose.yaml | 1 - kube/resources/worker.yaml | 5 -- 43 files changed, 141 insertions(+), 226 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java index 33f15d6f6f85e6..f95b3bc2aefd58 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java @@ -37,7 +37,10 @@ public void waitForTemporalNamespace() { // cache before continuing on with any additional configuration/bean creation. temporalService.blockingStub().describeNamespace(DescribeNamespaceRequest.newBuilder().setNamespace(temporalNamespace).build()); namespaceExists = true; - } catch (final StatusRuntimeException e) { + // This is to allow the configured namespace to be available in the Temporal + // cache before continuing on with any additional configuration/bean creation. + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + } catch (final InterruptedException | StatusRuntimeException e) { log.debug("Namespace '{}' does not exist yet. Re-checking...", temporalNamespace); try { Thread.sleep(TimeUnit.SECONDS.toMillis(5)); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index a46746b7a227cc..7b9a5d9b6fbaf1 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -203,8 +203,8 @@ private static JobOrchestrator getJobOrchestrator(final Configs configs, final String application, final FeatureFlags featureFlags) { return switch (application) { - case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, workerConfigs, processFactory, featureFlags); - case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, workerConfigs, processFactory); + case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, processFactory, featureFlags); + case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, processFactory); case DbtLauncherWorker.DBT -> new DbtJobOrchestrator(configs, workerConfigs, processFactory); case AsyncOrchestratorPodProcess.NO_OP -> new NoOpOrchestrator(); default -> null; diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java index c7fdf447660f98..b46a34d9acf7d7 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java @@ -58,9 +58,7 @@ public Optional runJob() throws Exception { Math.toIntExact(jobRunConfig.getAttemptId()), workerConfigs.getResourceRequirements(), new DbtTransformationRunner( - workerConfigs, processFactory, NormalizationRunnerFactory.create( - workerConfigs, destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION))); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java index 8dbcc6b8b53bf4..2427ccafe06b9a 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java @@ -10,7 +10,6 @@ import io.airbyte.config.NormalizationSummary; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.general.DefaultNormalizationWorker; import io.airbyte.workers.normalization.NormalizationRunnerFactory; @@ -26,12 +25,10 @@ public class NormalizationJobOrchestrator implements JobOrchestrator { private final Configs configs; - private final WorkerConfigs workerConfigs; private final ProcessFactory processFactory; - public NormalizationJobOrchestrator(final Configs configs, final WorkerConfigs workerConfigs, final ProcessFactory processFactory) { + public NormalizationJobOrchestrator(final Configs configs, final ProcessFactory processFactory) { this.configs = configs; - this.workerConfigs = workerConfigs; this.processFactory = processFactory; } @@ -59,7 +56,6 @@ public Optional runJob() throws Exception { jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), NormalizationRunnerFactory.create( - workerConfigs, destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION), diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java index f39a5ca7f37795..53d575ff2e2c90 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java @@ -15,7 +15,6 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.RecordSchemaValidator; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.WorkerUtils; @@ -40,16 +39,13 @@ public class ReplicationJobOrchestrator implements JobOrchestrator { private final ProcessFactory processFactory; - private final WorkerConfigs workerConfigs; private final Configs configs; private final FeatureFlags featureFlags; public ReplicationJobOrchestrator(final Configs configs, - final WorkerConfigs workerConfigs, final ProcessFactory processFactory, final FeatureFlags featureFlags) { this.configs = configs; - this.workerConfigs = workerConfigs; this.processFactory = processFactory; this.featureFlags = featureFlags; } @@ -98,7 +94,7 @@ public Optional runJob() throws Exception { final AirbyteSource airbyteSource = WorkerConstants.RESET_JOB_SOURCE_DOCKER_IMAGE_STUB.equals(sourceLauncherConfig.getDockerImage()) ? new EmptyAirbyteSource( featureFlags.useStreamCapableState()) - : new DefaultAirbyteSource(workerConfigs, sourceLauncher); + : new DefaultAirbyteSource(sourceLauncher); MetricClientFactory.initialize(MetricEmittingApps.WORKER); final MetricClient metricClient = MetricClientFactory.getMetricClient(); @@ -110,7 +106,7 @@ public Optional runJob() throws Exception { Math.toIntExact(jobRunConfig.getAttemptId()), airbyteSource, new NamespacingMapper(syncInput.getNamespaceDefinition(), syncInput.getNamespaceFormat(), syncInput.getPrefix()), - new DefaultAirbyteDestination(workerConfigs, destinationLauncher), + new DefaultAirbyteDestination(destinationLauncher), new AirbyteMessageTracker(), new RecordSchemaValidator(WorkerUtils.mapStreamNamesToSchemas(syncInput)), metricReporter); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 96ede81218d097..c503edd6e3234a 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -507,7 +507,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { if (normalizationFromSpec) { boolean normalizationRunnerFactorySupportsDestinationImage; try { - NormalizationRunnerFactory.create(workerConfigs, getImageName(), processFactory, NORMALIZATION_VERSION); + NormalizationRunnerFactory.create(getImageName(), processFactory, NORMALIZATION_VERSION); normalizationRunnerFactorySupportsDestinationImage = true; } catch (final IllegalStateException e) { normalizationRunnerFactorySupportsDestinationImage = false; @@ -781,8 +781,7 @@ public void testCustomDbtTransformations() throws Exception { // this test successfully and that we are able to convert a destination 'config.json' into a dbt // 'profiles.yml' // (we don't actually rely on normalization running anything else here though) - final DbtTransformationRunner runner = new DbtTransformationRunner(workerConfigs, processFactory, NormalizationRunnerFactory.create( - workerConfigs, + final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory, NormalizationRunnerFactory.create( getImageName(), processFactory, NORMALIZATION_VERSION)); @@ -857,8 +856,7 @@ void testCustomDbtTransformationsFailure() throws Exception { final JsonNode config = getConfig(); - final DbtTransformationRunner runner = new DbtTransformationRunner(workerConfigs, processFactory, NormalizationRunnerFactory.create( - workerConfigs, + final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory, NormalizationRunnerFactory.create( getImageName(), processFactory, NORMALIZATION_VERSION)); @@ -1056,7 +1054,7 @@ public void testSyncNotFailsWithNewFields() throws Exception { // Run sync and verify that all message were written without failing runSyncAndVerifyStateOutput(config, secondSyncMessagesWithNewFields, configuredCatalog, false); - var destinationOutput = retrieveRecords(testEnv, stream.getName(), getDefaultSchema(config), stream.getJsonSchema()); + final var destinationOutput = retrieveRecords(testEnv, stream.getName(), getDefaultSchema(config), stream.getJsonSchema()); // Remove state message secondSyncMessagesWithNewFields.removeIf(airbyteMessage -> airbyteMessage.getType().equals(Type.STATE)); assertEquals(secondSyncMessagesWithNewFields.size(), destinationOutput.size()); @@ -1090,20 +1088,20 @@ protected void assertNamespaceNormalization(final String testCaseId, private ConnectorSpecification runSpec() throws WorkerException { return new DefaultGetSpecWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) + new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot).getSpec(); } protected StandardCheckConnectionOutput runCheck(final JsonNode config) throws WorkerException { return new DefaultCheckConnectionWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) + new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) .run(new StandardCheckConnectionInput().withConnectionConfiguration(config), jobRoot).getCheckConnection(); } protected StandardCheckConnectionOutput.Status runCheckWithCatchedException(final JsonNode config) { try { final StandardCheckConnectionOutput standardCheckConnectionOutput = new DefaultCheckConnectionWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) + new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)) .run(new StandardCheckConnectionInput().withConnectionConfiguration(config), jobRoot).getCheckConnection(); return standardCheckConnectionOutput.getStatus(); } catch (final Exception e) { @@ -1114,7 +1112,7 @@ workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName( protected AirbyteDestination getDestination() { return new DefaultAirbyteDestination( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)); + new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null)); } protected void runSyncAndVerifyStateOutput(final JsonNode config, @@ -1171,7 +1169,6 @@ private List runSync( } final NormalizationRunner runner = NormalizationRunnerFactory.create( - workerConfigs, getImageName(), processFactory, NORMALIZATION_VERSION); diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java index ecea6f96332e6b..9505978062a1c3 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java @@ -127,28 +127,24 @@ public void tearDownInternal() throws Exception { protected ConnectorSpecification runSpec() throws WorkerException { return new DefaultGetSpecWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements())) .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot).getSpec(); } protected StandardCheckConnectionOutput runCheck() throws Exception { return new DefaultCheckConnectionWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements())) .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot).getCheckConnection(); } protected String runCheckAndGetStatusAsString(final JsonNode config) throws Exception { return new DefaultCheckConnectionWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements())) .run(new StandardCheckConnectionInput().withConnectionConfiguration(config), jobRoot).getCheckConnection().getStatus().toString(); } protected AirbyteCatalog runDiscover() throws Exception { return new DefaultDiscoverCatalogWorker( - workerConfigs, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements())) .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot).getDiscoverCatalog(); } @@ -156,7 +152,7 @@ protected AirbyteCatalog runDiscover() throws Exception { protected void checkEntrypointEnvVariable() throws Exception { final String entrypoint = EntrypointEnvChecker.getEntrypointEnvVariable( processFactory, - String.valueOf(JOB_ID), + JOB_ID, JOB_ATTEMPT, jobRoot, getImageName()); @@ -176,7 +172,7 @@ protected List runRead(final ConfiguredAirbyteCatalog catalog, f .withState(state == null ? null : new State().withState(state)) .withCatalog(catalog); - final AirbyteSource source = new DefaultAirbyteSource(workerConfigs, + final AirbyteSource source = new DefaultAirbyteSource( new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements())); final List messages = new ArrayList<>(); source.start(sourceConfig, jobRoot); @@ -229,7 +225,7 @@ private AirbyteSource prepareAirbyteSource(final ResourceRequirements resourceRe final var integrationLauncher = resourceRequirements == null ? new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements()) : new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, resourceRequirements); - return new DefaultAirbyteSource(workerConfigs, integrationLauncher); + return new DefaultAirbyteSource(integrationLauncher); } private static Map prepareResourceRequestMapBySystemProperties() { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java index d07dce680e2c99..b024cb99760c0e 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java @@ -12,11 +12,9 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.EnvConfigs; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerUtils; import java.io.IOException; import java.nio.file.Files; @@ -133,7 +131,7 @@ private Path runExecutableInternal(final Command cmd) throws IOException { LineGobbler.gobble(process.getErrorStream(), LOGGER::error); LineGobbler.gobble(process.getInputStream(), LOGGER::info); - WorkerUtils.gentleClose(new WorkerConfigs(new EnvConfigs()), process, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); final int exitCode = process.exitValue(); if (exitCode != 0) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java b/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java index 635a500f579332..1cbfae572a1348 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java @@ -181,7 +181,7 @@ private void initializeCommonDependencies() throws ExecutionException, Interrupt LogClientSingleton.getInstance().setWorkspaceMdc(workerEnvironment, logConfigs.orElseThrow(), LogClientSingleton.getInstance().getSchedulerLogsRoot(Path.of(workspaceRoot))); - if (WorkerEnvironment.KUBERNETES.equals(workerEnvironment)) { + if (environment.getActiveNames().contains(Environment.KUBERNETES)) { KubePortManagerSingleton.init(temporalWorkerPorts); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/ContainerOrchestratorConfig.java b/airbyte-workers/src/main/java/io/airbyte/workers/ContainerOrchestratorConfig.java index bf0e68e5ee095d..c3f29292e7cfbb 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/ContainerOrchestratorConfig.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/ContainerOrchestratorConfig.java @@ -4,6 +4,7 @@ package io.airbyte.workers; +import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.workers.general.DocumentStoreClient; import io.fabric8.kubernetes.client.KubernetesClient; @@ -15,4 +16,5 @@ public record ContainerOrchestratorConfig( String secretMountPath, String containerOrchestratorImage, String containerOrchestratorImagePullPolicy, - String googleApplicationCredentials) {} + String googleApplicationCredentials, + WorkerEnvironment workerEnvironment) {} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java index 7ecbb8d0d9da82..21d8d01218d5d6 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java @@ -5,7 +5,6 @@ package io.airbyte.workers; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.ConnectorJobOutput; import io.airbyte.config.ConnectorJobOutput.OutputType; import io.airbyte.config.FailureReason; @@ -39,14 +38,14 @@ public class WorkerUtils { private static final Logger LOGGER = LoggerFactory.getLogger(WorkerUtils.class); - public static void gentleClose(final WorkerConfigs workerConfigs, final Process process, final long timeout, final TimeUnit timeUnit) { + public static void gentleClose(final Process process, final long timeout, final TimeUnit timeUnit) { if (process == null) { return; } - if (workerConfigs.getWorkerEnvironment().equals(WorkerEnvironment.KUBERNETES)) { - LOGGER.debug("Gently closing process {}", process.info().commandLine().get()); + if (process.info() != null) { + process.info().commandLine().ifPresent(commandLine -> LOGGER.debug("Gently closing process {}", commandLine)); } try { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApplicationBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApplicationBeanFactory.java index 411a0c97bece86..c14c2431cfa975 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApplicationBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApplicationBeanFactory.java @@ -28,6 +28,7 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; +import io.micronaut.context.env.Environment; import io.micronaut.core.util.StringUtils; import jakarta.inject.Named; import jakarta.inject.Singleton; @@ -68,8 +69,8 @@ public TrackingStrategy trackingStrategy(@Value("${airbyte.tracking-strategy}") } @Singleton - public WorkerEnvironment workerEnvironment(@Value("${airbyte.worker.env}") final String workerEnv) { - return convertToEnum(workerEnv, WorkerEnvironment::valueOf, WorkerEnvironment.DOCKER); + public WorkerEnvironment workerEnvironment(final Environment environment) { + return environment.getActiveNames().contains(Environment.KUBERNETES) ? WorkerEnvironment.KUBERNETES : WorkerEnvironment.DOCKER; } @Singleton diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ContainerOrchestratorConfigBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ContainerOrchestratorConfigBeanFactory.java index aae4afc894371e..e77a5d73532645 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ContainerOrchestratorConfigBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ContainerOrchestratorConfigBeanFactory.java @@ -4,6 +4,7 @@ package io.airbyte.workers.config; +import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.storage.CloudStorageConfigs; import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.general.DocumentStoreClient; @@ -40,7 +41,8 @@ public ContainerOrchestratorConfig kubernetesContainerOrchestratorConfig( @Value("${airbyte.container.orchestrator.secret-mount-path}") final String containerOrchestratorSecretMountPath, @Value("${airbyte.container.orchestrator.secret-name}") final String containerOrchestratorSecretName, @Value("${google.application.credentials}") final String googleApplicationCredentials, - @Value("${airbyte.worker.job.kube.namespace}") final String namespace) { + @Value("${airbyte.worker.job.kube.namespace}") final String namespace, + final WorkerEnvironment workerEnvironment) { final var kubernetesClient = new DefaultKubernetesClient(); final DocumentStoreClient documentStoreClient = StateClients.create( @@ -55,7 +57,8 @@ public ContainerOrchestratorConfig kubernetesContainerOrchestratorConfig( containerOrchestratorSecretMountPath, StringUtils.isNotEmpty(containerOrchestratorImage) ? containerOrchestratorImage : "airbyte/container-orchestrator:" + airbyteVersion, containerOrchestratorImagePullPolicy, - googleApplicationCredentials); + googleApplicationCredentials, + workerEnvironment); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ProcessFactoryBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ProcessFactoryBeanFactory.java index 43bb43a04e0697..e3f499624fc46a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ProcessFactoryBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ProcessFactoryBeanFactory.java @@ -13,6 +13,7 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; +import io.micronaut.context.env.Environment; import io.micronaut.core.util.StringUtils; import jakarta.inject.Named; import jakarta.inject.Singleton; @@ -28,9 +29,8 @@ public class ProcessFactoryBeanFactory { @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^(?!kubernetes$).*") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(notEnv = Environment.KUBERNETES) @Named("checkProcessFactory") public ProcessFactory checkDockerProcessFactory( @Named("checkWorkerConfigs") final WorkerConfigs workerConfigs, @@ -49,9 +49,8 @@ public ProcessFactory checkDockerProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^kubernetes$") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(env = Environment.KUBERNETES) @Named("checkProcessFactory") public ProcessFactory checkKubernetesProcessFactory( @Named("checkWorkerConfigs") final WorkerConfigs workerConfigs, @@ -64,8 +63,7 @@ public ProcessFactory checkKubernetesProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^(?!kubernetes$).*") + @Requires(notEnv = Environment.KUBERNETES) @Named("defaultProcessFactory") public ProcessFactory defaultDockerProcessFactory( @Named("defaultWorkerConfigs") final WorkerConfigs workerConfigs, @@ -84,8 +82,7 @@ public ProcessFactory defaultDockerProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^kubernetes$") + @Requires(env = Environment.KUBERNETES) @Named("defaultProcessFactory") public ProcessFactory defaultKubernetesProcessFactory( @Named("defaultWorkerConfigs") final WorkerConfigs workerConfigs, @@ -98,9 +95,8 @@ public ProcessFactory defaultKubernetesProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^(?!kubernetes$).*") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(notEnv = Environment.KUBERNETES) @Named("discoverProcessFactory") public ProcessFactory discoverDockerProcessFactory( @Named("discoverWorkerConfigs") final WorkerConfigs workerConfigs, @@ -119,9 +115,8 @@ public ProcessFactory discoverDockerProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^kubernetes$") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(env = Environment.KUBERNETES) @Named("discoverProcessFactory") public ProcessFactory discoverKubernetesProcessFactory( @Named("discoverWorkerConfigs") final WorkerConfigs workerConfigs, @@ -134,8 +129,7 @@ public ProcessFactory discoverKubernetesProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^(?!kubernetes$).*") + @Requires(notEnv = Environment.KUBERNETES) @Named("replicationProcessFactory") public ProcessFactory replicationDockerProcessFactory( @Named("replicationWorkerConfigs") final WorkerConfigs workerConfigs, @@ -154,8 +148,7 @@ public ProcessFactory replicationDockerProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^kubernetes$") + @Requires(env = Environment.KUBERNETES) @Named("replicationProcessFactory") public ProcessFactory replicationKubernetesProcessFactory( @Named("replicationWorkerConfigs") final WorkerConfigs workerConfigs, @@ -168,9 +161,8 @@ public ProcessFactory replicationKubernetesProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^(?!kubernetes$).*") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(notEnv = Environment.KUBERNETES) @Named("specProcessFactory") public ProcessFactory specDockerProcessFactory( @Named("specWorkerConfigs") final WorkerConfigs workerConfigs, @@ -189,9 +181,8 @@ public ProcessFactory specDockerProcessFactory( } @Singleton - @Requires(property = "airbyte.worker.env", - pattern = "(?i)^kubernetes$") @Requires(env = WorkerMode.CONTROL_PLANE) + @Requires(env = Environment.KUBERNETES) @Named("specProcessFactory") public ProcessFactory specKubernetesProcessFactory( @Named("specWorkerConfigs") final WorkerConfigs workerConfigs, @@ -217,16 +208,15 @@ private ProcessFactory createDockerProcessFactory(final WorkerConfigs workerConf dockerNetwork); } - private ProcessFactory createKubernetesProcessFactory( - final WorkerConfigs workerConfigs, - final String kuberenetesNamespace, + private ProcessFactory createKubernetesProcessFactory(final WorkerConfigs workerConfigs, + final String kubernetesNamespace, final Integer serverPort) throws UnknownHostException { final KubernetesClient fabricClient = new DefaultKubernetesClient(); final String localIp = InetAddress.getLocalHost().getHostAddress(); final String kubeHeartbeatUrl = localIp + ":" + serverPort; return new KubeProcessFactory(workerConfigs, - kuberenetesNamespace, + kubernetesNamespace, fabricClient, kubeHeartbeatUrl, false); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/WorkerConfigurationBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/WorkerConfigurationBeanFactory.java index 1e476a96948fa5..a943cc21155347 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/WorkerConfigurationBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/WorkerConfigurationBeanFactory.java @@ -15,6 +15,7 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; +import io.micronaut.context.env.Environment; import jakarta.inject.Named; import jakarta.inject.Singleton; import java.util.List; @@ -36,8 +37,10 @@ public class WorkerConfigurationBeanFactory { private static final String AIRBYTE_ROLE = "AIRBYTE_ROLE"; private static final String AIRBYTE_VERSION = "AIRBYTE_VERSION"; private static final String DEPLOYMENT_MODE = "DEPLOYMENT_MODE"; + private static final String DOCKER = "DOCKER"; private static final String JOB_DEFAULT_ENV_PREFIX = "JOB_DEFAULT_ENV_"; - private static final String WORKER_ENVIRONMENT = "WORKER_ENVIRONMENT"; + private static final String KUBERNETES = "KUBERNETES"; + public static final String WORKER_ENVIRONMENT = "WORKER_ENVIRONMENT"; @Singleton @Named("checkJobKubeAnnotations") @@ -93,7 +96,7 @@ public Map jobDefaultEnvMap( @Value("${airbyte.role}") final String airbyteRole, @Value("${airbyte.version}") final String airbyteVersion, final DeploymentMode deploymentMode, - final WorkerEnvironment workerEnvironment) { + final Environment environment) { final Map envMap = System.getenv(); final Map jobPrefixedEnvMap = envMap.keySet().stream() .filter(key -> key.startsWith(JOB_DEFAULT_ENV_PREFIX)) @@ -101,7 +104,7 @@ public Map jobDefaultEnvMap( final Map jobSharedEnvMap = Map.of(AIRBYTE_ROLE, airbyteRole, AIRBYTE_VERSION, airbyteVersion, DEPLOYMENT_MODE, deploymentMode.name(), - WORKER_ENVIRONMENT, workerEnvironment.name()); + WORKER_ENVIRONMENT, environment.getActiveNames().contains(Environment.KUBERNETES) ? KUBERNETES : DOCKER); return MoreMaps.merge(jobPrefixedEnvMap, jobSharedEnvMap); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java index cd3c6fd31c1870..df0ff080715139 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java @@ -14,7 +14,6 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.OperatorDbt; import io.airbyte.config.ResourceRequirements; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.normalization.NormalizationRunner; @@ -38,15 +37,12 @@ public class DbtTransformationRunner implements AutoCloseable { .setLogPrefix("dbt") .setPrefixColor(Color.PURPLE_BACKGROUND); - private final WorkerConfigs workerConfigs; private final ProcessFactory processFactory; private final NormalizationRunner normalizationRunner; private Process process = null; - public DbtTransformationRunner(final WorkerConfigs workerConfigs, - final ProcessFactory processFactory, + public DbtTransformationRunner(final ProcessFactory processFactory, final NormalizationRunner normalizationRunner) { - this.workerConfigs = workerConfigs; this.processFactory = processFactory; this.normalizationRunner = normalizationRunner; } @@ -135,7 +131,7 @@ public void close() throws Exception { } LOGGER.debug("Closing dbt transformation process"); - WorkerUtils.gentleClose(workerConfigs, process, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); if (process.isAlive() || process.exitValue() != 0) { throw new WorkerException("Dbt transformation process wasn't successful"); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java index ee188f4051e74f..c29af2c84a969c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java @@ -16,7 +16,6 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; @@ -38,22 +37,19 @@ public class DefaultCheckConnectionWorker implements CheckConnectionWorker { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultCheckConnectionWorker.class); - private final WorkerConfigs workerConfigs; private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; private Process process; - public DefaultCheckConnectionWorker(final WorkerConfigs workerConfigs, - final IntegrationLauncher integrationLauncher, + public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) { - this.workerConfigs = workerConfigs; this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; } - public DefaultCheckConnectionWorker(final WorkerConfigs workerConfigs, final IntegrationLauncher integrationLauncher) { - this(workerConfigs, integrationLauncher, new DefaultAirbyteStreamFactory()); + public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLauncher) { + this(integrationLauncher, new DefaultAirbyteStreamFactory()); } @Override @@ -73,7 +69,7 @@ public ConnectorJobOutput run(final StandardCheckConnectionInput input, final Pa messagesByType = streamFactory.create(IOs.newBufferedReader(stdout)) .collect(Collectors.groupingBy(AirbyteMessage::getType)); - WorkerUtils.gentleClose(workerConfigs, process, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); } final int exitCode = process.exitValue(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java index 474d206137f398..92021ae9de49aa 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java @@ -13,7 +13,8 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.workers.*; +import io.airbyte.workers.WorkerConstants; +import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteStreamFactory; @@ -34,22 +35,19 @@ public class DefaultDiscoverCatalogWorker implements DiscoverCatalogWorker { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultDiscoverCatalogWorker.class); private static final int TEMPORAL_MESSAGE_LIMIT_MB = 1024 * 1024 * 4; - private final WorkerConfigs workerConfigs; private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; private volatile Process process; - public DefaultDiscoverCatalogWorker(final WorkerConfigs workerConfigs, - final IntegrationLauncher integrationLauncher, + public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) { - this.workerConfigs = workerConfigs; this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; } - public DefaultDiscoverCatalogWorker(final WorkerConfigs workerConfigs, final IntegrationLauncher integrationLauncher) { - this(workerConfigs, integrationLauncher, new DefaultAirbyteStreamFactory()); + public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLauncher) { + this(integrationLauncher, new DefaultAirbyteStreamFactory()); } @Override @@ -68,7 +66,7 @@ public ConnectorJobOutput run(final StandardDiscoverCatalogInput discoverSchemaI messagesByType = streamFactory.create(IOs.newBufferedReader(stdout)) .collect(Collectors.groupingBy(AirbyteMessage::getType)); - WorkerUtils.gentleClose(workerConfigs, process, 30, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 30, TimeUnit.MINUTES); } final Optional catalog = messagesByType diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java index 745b1ccbaf370c..cf6fbbb417a759 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java @@ -12,7 +12,6 @@ import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteStreamFactory; @@ -33,22 +32,19 @@ public class DefaultGetSpecWorker implements GetSpecWorker { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultGetSpecWorker.class); - private final WorkerConfigs workerConfigs; private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; private Process process; - public DefaultGetSpecWorker(final WorkerConfigs workerConfigs, - final IntegrationLauncher integrationLauncher, + public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) { - this.workerConfigs = workerConfigs; this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; } - public DefaultGetSpecWorker(final WorkerConfigs workerConfigs, final IntegrationLauncher integrationLauncher) { - this(workerConfigs, integrationLauncher, new DefaultAirbyteStreamFactory()); + public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher) { + this(integrationLauncher, new DefaultAirbyteStreamFactory()); } @Override @@ -67,7 +63,7 @@ public ConnectorJobOutput run(final JobGetSpecConfig config, final Path jobRoot) // this. // retrieving spec should generally be instantaneous, but since docker images might not be pulled // it could take a while longer depending on internet conditions as well. - WorkerUtils.gentleClose(workerConfigs, process, 30, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 30, TimeUnit.MINUTES); } final Optional spec = messagesByType diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java index a3c527dd226af0..87b1a7f79c114a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java @@ -15,7 +15,6 @@ import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; @@ -38,7 +37,6 @@ public class DefaultAirbyteDestination implements AirbyteDestination { .setLogPrefix("destination") .setPrefixColor(Color.YELLOW_BACKGROUND); - private final WorkerConfigs workerConfigs; private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; @@ -49,15 +47,13 @@ public class DefaultAirbyteDestination implements AirbyteDestination { private Iterator messageIterator = null; private Integer exitValue = null; - public DefaultAirbyteDestination(final WorkerConfigs workerConfigs, final IntegrationLauncher integrationLauncher) { - this(workerConfigs, integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER)); + public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher) { + this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER)); } - public DefaultAirbyteDestination(final WorkerConfigs workerConfigs, - final IntegrationLauncher integrationLauncher, + public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) { - this.workerConfigs = workerConfigs; this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; } @@ -112,7 +108,7 @@ public void close() throws Exception { } LOGGER.debug("Closing destination process"); - WorkerUtils.gentleClose(workerConfigs, destinationProcess, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(destinationProcess, 1, TimeUnit.MINUTES); if (destinationProcess.isAlive() || getExitValue() != 0) { final String message = destinationProcess.isAlive() ? "Destination has not terminated " : "Destination process exit with code " + getExitValue(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java index e618286d4ac43c..5e8415fc96d49d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java @@ -16,7 +16,6 @@ import io.airbyte.config.WorkerSourceConfig; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; @@ -41,7 +40,6 @@ public class DefaultAirbyteSource implements AirbyteSource { .setLogPrefix("source") .setPrefixColor(Color.BLUE_BACKGROUND); - private final WorkerConfigs workerConfigs; private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; private final HeartbeatMonitor heartbeatMonitor; @@ -51,17 +49,15 @@ public class DefaultAirbyteSource implements AirbyteSource { private Integer exitValue = null; private final boolean logConnectorMessages = new EnvVariableFeatureFlags().logConnectorMessages(); - public DefaultAirbyteSource(final WorkerConfigs workerConfigs, final IntegrationLauncher integrationLauncher) { - this(workerConfigs, integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER), + public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) { + this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION)); } @VisibleForTesting - DefaultAirbyteSource(final WorkerConfigs workerConfigs, - final IntegrationLauncher integrationLauncher, + DefaultAirbyteSource(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory, final HeartbeatMonitor heartbeatMonitor) { - this.workerConfigs = workerConfigs; this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; this.heartbeatMonitor = heartbeatMonitor; @@ -130,7 +126,6 @@ public void close() throws Exception { LOGGER.debug("Closing source process"); WorkerUtils.gentleClose( - workerConfigs, sourceProcess, GRACEFUL_SHUTDOWN_DURATION.toMillis(), TimeUnit.MILLISECONDS); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 5d4141bd004c84..3db73387fec0c7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -23,7 +23,6 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteTraceMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; @@ -47,7 +46,6 @@ public class DefaultNormalizationRunner implements NormalizationRunner { .setLogPrefix("normalization") .setPrefixColor(Color.GREEN_BACKGROUND); - private final WorkerConfigs workerConfigs; private final DestinationType destinationType; private final ProcessFactory processFactory; private final String normalizationImageName; @@ -69,11 +67,9 @@ public enum DestinationType { TIDB } - public DefaultNormalizationRunner(final WorkerConfigs workerConfigs, - final DestinationType destinationType, + public DefaultNormalizationRunner(final DestinationType destinationType, final ProcessFactory processFactory, final String normalizationImageName) { - this.workerConfigs = workerConfigs; this.destinationType = destinationType; this.processFactory = processFactory; this.normalizationImageName = normalizationImageName; @@ -199,7 +195,7 @@ public void close() throws Exception { } LOGGER.debug("Closing normalization process"); - WorkerUtils.gentleClose(workerConfigs, process, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); if (process.isAlive() || process.exitValue() != 0) { throw new WorkerException("Normalization process wasn't successful"); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index eb9a8ae74204bc..9a5c1b9e6373a4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -5,7 +5,6 @@ package io.airbyte.workers.normalization; import com.google.common.collect.ImmutableMap; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; import io.airbyte.workers.process.ProcessFactory; import java.util.Map; @@ -37,13 +36,11 @@ public class NormalizationRunnerFactory { .put("airbyte/destination-tidb", ImmutablePair.of("airbyte/normalization-tidb", DestinationType.TIDB)) .build(); - public static NormalizationRunner create(final WorkerConfigs workerConfigs, - final String connectorImageName, + public static NormalizationRunner create(final String connectorImageName, final ProcessFactory processFactory, final String normalizationVersion) { final var valuePair = getNormalizationInfoForConnector(connectorImageName); return new DefaultNormalizationRunner( - workerConfigs, valuePair.getRight(), processFactory, String.format("%s:%s", valuePair.getLeft(), normalizationVersion)); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java index e3d14f6bb54c69..743a429a772d88 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java @@ -180,7 +180,7 @@ boolean checkImageExists(final String imageName) throws WorkerException { LineGobbler.gobble(process.getErrorStream(), LOGGER::error); LineGobbler.gobble(process.getInputStream(), LOGGER::info); - WorkerUtils.gentleClose(workerConfigs, process, 10, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 10, TimeUnit.MINUTES); if (process.isAlive()) { throw new WorkerException("Process to check if image exists is stuck. Exiting."); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java index ee19ef5ad55352..b53e6021199929 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java @@ -106,7 +106,7 @@ private CheckedSupplier processFactory, workerConfigs.getResourceRequirements()); - return new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher); + return new DefaultCheckConnectionWorker(integrationLauncher); }; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java index 15bdbb4e74cf1a..c1256b93a009f1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java @@ -99,7 +99,7 @@ private CheckedSupplier new AirbyteIntegrationLauncher(launcherConfig.getJobId(), launcherConfig.getAttemptId().intValue(), launcherConfig.getDockerImage(), processFactory, workerConfigs.getResourceRequirements()); final AirbyteStreamFactory streamFactory = new DefaultAirbyteStreamFactory(); - return new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, streamFactory); + return new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); }; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/spec/SpecActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/spec/SpecActivityImpl.java index 0f3eb998bf96fb..5cdb899fdfbacf 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/spec/SpecActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/spec/SpecActivityImpl.java @@ -89,7 +89,7 @@ private CheckedSupplier, Exception> processFactory, workerConfigs.getResourceRequirements()); - return new DefaultGetSpecWorker(workerConfigs, integrationLauncher); + return new DefaultGetSpecWorker(integrationLauncher); }; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index b23e5344bdaef1..1ee47429ae79f7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -130,9 +130,7 @@ private CheckedSupplier, Exception> getLegacyWork Math.toIntExact(jobRunConfig.getAttemptId()), resourceRequirements, new DbtTransformationRunner( - workerConfigs, processFactory, NormalizationRunnerFactory.create( - workerConfigs, destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION))); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java index eb07bfbc452d44..7b1d848ad958f5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/LauncherWorker.java @@ -13,6 +13,7 @@ import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.Worker; +import io.airbyte.workers.config.WorkerConfigurationBeanFactory; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.process.AsyncKubePodStatus; import io.airbyte.workers.process.AsyncOrchestratorPodProcess; @@ -102,6 +103,9 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + // Manually add the worker environment to the env var map + envMap.put(WorkerConfigurationBeanFactory.WORKER_ENVIRONMENT, containerOrchestratorConfig.workerEnvironment().name()); + final Map fileMap = new HashMap<>(additionalFileMap); fileMap.putAll(Map.of( OrchestratorConstants.INIT_FILE_APPLICATION, application, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index b5119ca6c39dcb..cfa4a46593342c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -105,7 +105,7 @@ public NormalizationSummary normalize(final JobRunConfig jobRunConfig, workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig, () -> context); } else { - workerFactory = getLegacyWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig); + workerFactory = getLegacyWorkerFactory(destinationLauncherConfig, jobRunConfig); } final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( @@ -132,14 +132,12 @@ public NormalizationInput generateNormalizationInput(final StandardSyncInput syn } private CheckedSupplier, Exception> getLegacyWorkerFactory( - final WorkerConfigs workerConfigs, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig) { return () -> new DefaultNormalizationWorker( jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), NormalizationRunnerFactory.create( - workerConfigs, destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION), diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index 945e4a63cd694a..955f18bc6bb679 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -30,7 +30,6 @@ import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.Worker; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.WorkerUtils; @@ -64,7 +63,6 @@ public class ReplicationActivityImpl implements ReplicationActivity { private static final int MAX_TEMPORAL_MESSAGE_SIZE = 2 * 1024 * 1024; private final Optional containerOrchestratorConfig; - private final WorkerConfigs workerConfigs; private final ProcessFactory processFactory; private final SecretsHydrator secretsHydrator; private final Path workspaceRoot; @@ -78,7 +76,6 @@ public class ReplicationActivityImpl implements ReplicationActivity { private final AirbyteApiClient airbyteApiClient; public ReplicationActivityImpl(@Named("containerOrchestratorConfig") final Optional containerOrchestratorConfig, - @Named("replicationWorkerConfigs") final WorkerConfigs workerConfigs, @Named("replicationProcessFactory") final ProcessFactory processFactory, final SecretsHydrator secretsHydrator, @Named("workspaceRoot") final Path workspaceRoot, @@ -91,7 +88,6 @@ public ReplicationActivityImpl(@Named("containerOrchestratorConfig") final Optio final TemporalUtils temporalUtils, final AirbyteApiClient airbyteApiClient) { this.containerOrchestratorConfig = containerOrchestratorConfig; - this.workerConfigs = workerConfigs; this.processFactory = processFactory; this.secretsHydrator = secretsHydrator; this.workspaceRoot = workspaceRoot; @@ -215,7 +211,7 @@ private CheckedSupplier, Exception> final AirbyteSource airbyteSource = WorkerConstants.RESET_JOB_SOURCE_DOCKER_IMAGE_STUB.equals(sourceLauncherConfig.getDockerImage()) ? new EmptyAirbyteSource(featureFlags.useStreamCapableState()) - : new DefaultAirbyteSource(workerConfigs, sourceLauncher); + : new DefaultAirbyteSource(sourceLauncher); MetricClientFactory.initialize(MetricEmittingApps.WORKER); final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); @@ -225,7 +221,7 @@ private CheckedSupplier, Exception> Math.toIntExact(jobRunConfig.getAttemptId()), airbyteSource, new NamespacingMapper(syncInput.getNamespaceDefinition(), syncInput.getNamespaceFormat(), syncInput.getPrefix()), - new DefaultAirbyteDestination(workerConfigs, destinationLauncher), + new DefaultAirbyteDestination(destinationLauncher), new AirbyteMessageTracker(), new RecordSchemaValidator(WorkerUtils.mapStreamNamesToSchemas(syncInput)), metricReporter); diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index 54b95f0b2f3bde..465283f0f0f7a0 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -82,10 +82,9 @@ airbyte: value: ${AIRBYTE_API_AUTH_HEADER_VALUE:} host: ${INTERNAL_API_HOST} local: - docker-mount: ${LOCAL_DOCKER_MOUNT} + docker-mount: ${LOCAL_DOCKER_MOUNT:} root: ${LOCAL_ROOT} worker: - env: ${WORKER_ENVIRONMENT:DOCKER} check: enabled: ${SHOULD_RUN_CHECK_CONNECTION_WORKFLOWS:true} kube: diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java index 38630c595283c6..0095be0c6b0a40 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java @@ -20,7 +20,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConnectorJobOutput; import io.airbyte.config.ConnectorJobOutput.OutputType; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.FailureReason; import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardCheckConnectionOutput; @@ -28,7 +27,6 @@ import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteMessageUtils; @@ -48,7 +46,6 @@ class DefaultCheckConnectionWorkerTest { private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); private static final JsonNode CREDS = Jsons.jsonNode(ImmutableMap.builder().put("apiKey", "123").build()); - private WorkerConfigs workerConfigs; private Path jobRoot; private StandardCheckConnectionInput input; private IntegrationLauncher integrationLauncher; @@ -59,7 +56,6 @@ class DefaultCheckConnectionWorkerTest { @BeforeEach void setup() throws IOException, WorkerException { - workerConfigs = WorkerConfigs.buildCheckWorkerConfigs(new EnvConfigs()); input = new StandardCheckConnectionInput().withConnectionConfiguration(CREDS); jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), ""); @@ -92,7 +88,7 @@ void testEnums() { @Test void testSuccessfulConnection() throws WorkerException { - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, successStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, successStreamFactory); final ConnectorJobOutput output = worker.run(input, jobRoot); assertEquals(output.getOutputType(), OutputType.CHECK_CONNECTION); @@ -105,7 +101,7 @@ void testSuccessfulConnection() throws WorkerException { @Test void testFailedConnection() throws WorkerException { - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, failureStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, failureStreamFactory); final ConnectorJobOutput output = worker.run(input, jobRoot); assertEquals(output.getOutputType(), OutputType.CHECK_CONNECTION); @@ -120,7 +116,7 @@ void testFailedConnection() throws WorkerException { void testProcessFail() { when(process.exitValue()).thenReturn(1); - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, failureStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, failureStreamFactory); assertThrows(WorkerException.class, () -> worker.run(input, jobRoot)); } @@ -128,7 +124,7 @@ void testProcessFail() { void testProcessFailWithTraceMessage() throws WorkerException { when(process.exitValue()).thenReturn(1); - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, traceMessageStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, traceMessageStreamFactory); final ConnectorJobOutput output = worker.run(input, jobRoot); assertEquals(output.getOutputType(), OutputType.CHECK_CONNECTION); @@ -142,14 +138,14 @@ void testProcessFailWithTraceMessage() throws WorkerException { void testExceptionThrownInRun() throws WorkerException { doThrow(new RuntimeException()).when(integrationLauncher).check(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDS)); - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, failureStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, failureStreamFactory); assertThrows(WorkerException.class, () -> worker.run(input, jobRoot)); } @Test void testCancel() throws WorkerException { - final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(workerConfigs, integrationLauncher, successStreamFactory); + final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, successStreamFactory); worker.run(input, jobRoot); worker.cancel(); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java index 5d99f6175756f0..73da1890e71f6d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java @@ -25,7 +25,6 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.ConnectorJobOutput; import io.airbyte.config.ConnectorJobOutput.OutputType; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.FailureReason; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.protocol.models.AirbyteCatalog; @@ -34,7 +33,6 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteMessageUtils; @@ -64,7 +62,6 @@ class DefaultDiscoverCatalogWorkerTest { Field.of(COLUMN_NAME, JsonSchemaType.STRING), Field.of(COLUMN_AGE, JsonSchemaType.NUMBER)))); - private WorkerConfigs workerConfigs; private Path jobRoot; private IntegrationLauncher integrationLauncher; private Process process; @@ -72,7 +69,6 @@ class DefaultDiscoverCatalogWorkerTest { @BeforeEach void setup() throws Exception { - workerConfigs = new WorkerConfigs(new EnvConfigs()); jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), ""); integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS); process = mock(Process.class); @@ -90,7 +86,7 @@ void setup() throws Exception { @SuppressWarnings("BusyWait") @Test void testDiscoverSchema() throws Exception { - final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, streamFactory); + final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); final ConnectorJobOutput output = worker.run(INPUT, jobRoot); assertNull(output.getFailureReason()); @@ -111,7 +107,7 @@ void testDiscoverSchema() throws Exception { void testDiscoverSchemaProcessFail() throws Exception { when(process.exitValue()).thenReturn(1); - final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, streamFactory); + final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot)); Assertions.assertTimeout(Duration.ofSeconds(5), () -> { @@ -131,7 +127,7 @@ void testDiscoverSchemaProcessFailWithTraceMessage() throws Exception { when(process.exitValue()).thenReturn(1); - final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, traceStreamFactory); + final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, traceStreamFactory); final ConnectorJobOutput output = worker.run(INPUT, jobRoot); assertEquals(OutputType.DISCOVER_CATALOG, output.getOutputType()); assertNull(output.getDiscoverCatalog()); @@ -154,13 +150,13 @@ void testDiscoverSchemaException() throws WorkerException { when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS))) .thenThrow(new RuntimeException()); - final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, streamFactory); + final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot)); } @Test void testCancel() throws WorkerException { - final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(workerConfigs, integrationLauncher, streamFactory); + final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); worker.run(INPUT, jobRoot); worker.cancel(); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java index a98ac3ecd7dcfa..2b06a7669fabea 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java @@ -20,13 +20,11 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.ConnectorJobOutput; import io.airbyte.config.ConnectorJobOutput.OutputType; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.FailureReason; import io.airbyte.config.JobGetSpecConfig; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteMessageUtils; import io.airbyte.workers.process.IntegrationLauncher; @@ -57,7 +55,7 @@ void setup() throws IOException, WorkerException { when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0])); when(integrationLauncher.spec(jobRoot)).thenReturn(process); - worker = new DefaultGetSpecWorker(new WorkerConfigs(new EnvConfigs()), integrationLauncher); + worker = new DefaultGetSpecWorker(integrationLauncher); } @Test diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java index ed0131e364007c..9a2eef1cf3a636 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java @@ -21,13 +21,11 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.config.Configs.WorkerEnvironment; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.workers.TestConfigHelpers; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; @@ -76,7 +74,6 @@ class DefaultAirbyteDestinationTest { } } - private WorkerConfigs workerConfigs; private Path jobRoot; private IntegrationLauncher integrationLauncher; private Process process; @@ -85,7 +82,6 @@ class DefaultAirbyteDestinationTest { @BeforeEach void setup() throws IOException, WorkerException { - workerConfigs = new WorkerConfigs(new EnvConfigs()); jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), JOB_ROOT_PREFIX); process = mock(Process.class); @@ -123,7 +119,7 @@ void tearDown() throws IOException { @SuppressWarnings("BusyWait") @Test void testSuccessfulLifecycle() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(workerConfigs, integrationLauncher, streamFactory); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory); destination.start(DESTINATION_CONFIG, jobRoot); final AirbyteMessage recordMessage = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"); @@ -162,7 +158,7 @@ void testSuccessfulLifecycle() throws Exception { @Test void testTaggedLogs() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(workerConfigs, integrationLauncher, streamFactory); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory); destination.start(DESTINATION_CONFIG, jobRoot); final AirbyteMessage recordMessage = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"); @@ -190,7 +186,7 @@ void testTaggedLogs() throws Exception { @Test void testCloseNotifiesLifecycle() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(workerConfigs, integrationLauncher); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher); destination.start(DESTINATION_CONFIG, jobRoot); verify(outputStream, never()).close(); @@ -202,7 +198,7 @@ void testCloseNotifiesLifecycle() throws Exception { @Test void testNonzeroExitCodeThrowsException() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(workerConfigs, integrationLauncher); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher); destination.start(DESTINATION_CONFIG, jobRoot); when(process.isAlive()).thenReturn(false); @@ -212,7 +208,7 @@ void testNonzeroExitCodeThrowsException() throws Exception { @Test void testGetExitValue() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(workerConfigs, integrationLauncher); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher); destination.start(DESTINATION_CONFIG, jobRoot); when(process.isAlive()).thenReturn(false); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteSourceTest.java index 976a5c3086d85d..38fcf48daf1ded 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteSourceTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/DefaultAirbyteSourceTest.java @@ -21,7 +21,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.config.Configs.WorkerEnvironment; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.State; import io.airbyte.config.WorkerSourceConfig; import io.airbyte.config.helpers.LogClientSingleton; @@ -31,7 +30,6 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.process.IntegrationLauncher; @@ -90,7 +88,6 @@ class DefaultAirbyteSourceTest { } } - private WorkerConfigs workerConfigs; private Path jobRoot; private IntegrationLauncher integrationLauncher; private Process process; @@ -99,7 +96,6 @@ class DefaultAirbyteSourceTest { @BeforeEach void setup() throws IOException, WorkerException { - workerConfigs = new WorkerConfigs(new EnvConfigs()); jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "test"); integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS); @@ -140,7 +136,7 @@ void testSuccessfulLifecycle() throws Exception { when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false); - final AirbyteSource source = new DefaultAirbyteSource(workerConfigs, integrationLauncher, streamFactory, heartbeatMonitor); + final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor); source.start(SOURCE_CONFIG, jobRoot); final List messages = Lists.newArrayList(); @@ -175,7 +171,7 @@ void testTaggedLogs() throws Exception { when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false); - final AirbyteSource source = new DefaultAirbyteSource(workerConfigs, integrationLauncher, streamFactory, + final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor); source.start(SOURCE_CONFIG, jobRoot); @@ -201,7 +197,7 @@ void testTaggedLogs() throws Exception { @Test void testNonzeroExitCodeThrows() throws Exception { - final AirbyteSource tap = new DefaultAirbyteSource(workerConfigs, integrationLauncher, streamFactory, heartbeatMonitor); + final AirbyteSource tap = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor); tap.start(SOURCE_CONFIG, jobRoot); when(process.exitValue()).thenReturn(1); @@ -211,7 +207,7 @@ void testNonzeroExitCodeThrows() throws Exception { @Test void testGetExitValue() throws Exception { - final AirbyteSource source = new DefaultAirbyteSource(workerConfigs, integrationLauncher, streamFactory, heartbeatMonitor); + final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor); source.start(SOURCE_CONFIG, jobRoot); when(process.isAlive()).thenReturn(false); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java index e6963918ccad73..6b182f2c656eb4 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java @@ -110,7 +110,7 @@ public void tearDown() throws IOException { @Test void test() throws Exception { final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); when(process.exitValue()).thenReturn(0); @@ -122,7 +122,7 @@ void test() throws Exception { void testLog() throws Exception { final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); when(process.exitValue()).thenReturn(0); @@ -145,7 +145,7 @@ void testClose() throws Exception { when(process.isAlive()).thenReturn(true).thenReturn(false); final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()); runner.close(); @@ -158,7 +158,7 @@ void testFailure() throws Exception { when(process.exitValue()).thenReturn(1); final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); @@ -171,16 +171,16 @@ void testFailure() throws Exception { void testFailureWithTraceMessage() throws Exception { when(process.exitValue()).thenReturn(1); - String errorTraceString = """ - {"type": "TRACE", "trace": { - "type": "ERROR", "emitted_at": 123.0, "error": { - "message": "Something went wrong in normalization.", "internal_message": "internal msg", - "stack_trace": "abc.xyz", "failure_type": "system_error"}}} - """.replace("\n", ""); + final String errorTraceString = """ + {"type": "TRACE", "trace": { + "type": "ERROR", "emitted_at": 123.0, "error": { + "message": "Something went wrong in normalization.", "internal_message": "internal msg", + "stack_trace": "abc.xyz", "failure_type": "system_error"}}} + """.replace("\n", ""); when(process.getInputStream()).thenReturn(new ByteArrayInputStream(errorTraceString.getBytes(StandardCharsets.UTF_8))); final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); @@ -195,19 +195,19 @@ void testFailureWithTraceMessage() throws Exception { void testFailureWithDbtError() throws Exception { when(process.exitValue()).thenReturn(1); - String dbtErrorString = """ - [info ] [MainThread]: Completed with 1 error and 0 warnings: - [info ] [MainThread]: - [error] [MainThread]: Database Error in model xyz (models/generated/airbyte_incremental/abc/xyz.sql) - [error] [MainThread]: 1292 (22007): Truncated incorrect DOUBLE value: 'ABC' - [error] [MainThread]: compiled SQL at ../build/run/airbyte_utils/models/generated/airbyte_incremental/abc/xyz.sql - [info ] [MainThread]: - [info ] [MainThread]: Done. PASS=1 WARN=0 ERROR=1 SKIP=0 TOTAL=2 - """; + final String dbtErrorString = """ + [info ] [MainThread]: Completed with 1 error and 0 warnings: + [info ] [MainThread]: + [error] [MainThread]: Database Error in model xyz (models/generated/airbyte_incremental/abc/xyz.sql) + [error] [MainThread]: 1292 (22007): Truncated incorrect DOUBLE value: 'ABC' + [error] [MainThread]: compiled SQL at ../build/run/airbyte_utils/models/generated/airbyte_incremental/abc/xyz.sql + [info ] [MainThread]: + [info ] [MainThread]: Done. PASS=1 WARN=0 ERROR=1 SKIP=0 TOTAL=2 + """; when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); @@ -222,14 +222,14 @@ void testFailureWithDbtError() throws Exception { void testFailureWithDbtErrorJsonFormat() throws Exception { when(process.exitValue()).thenReturn(1); - String dbtErrorString = + final String dbtErrorString = """ {"code": "Q035", "data": {"description": "table model public.start_products", "execution_time": 0.1729569435119629, "index": 1, "status": "error", "total": 2}, "invocation_id": "6ada8ee5-11c1-4239-8bd0-7e45178217c5", "level": "error", "log_version": 1, "msg": "1 of 2 ERROR creating table model public.start_products................................................................. [\\u001b[31mERROR\\u001b[0m in 0.17s]", "node_info": {"materialized": "table", "node_finished_at": null, "node_name": "start_products", "node_path": "generated/airbyte_incremental/public/start_products.sql", "node_started_at": "2022-07-18T15:04:27.036328", "node_status": "compiling", "resource_type": "model", "type": "node_status", "unique_id": "model.airbyte_utils.start_products"}, "pid": 14, "thread_name": "Thread-1", "ts": "2022-07-18T15:04:27.215077Z", "type": "log_line"} """; when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); final NormalizationRunner runner = - new DefaultNormalizationRunner(workerConfigs, DestinationType.BIGQUERY, processFactory, + new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java index 5f35c6c694c63e..4db4152b7373f0 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java @@ -8,8 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; -import io.airbyte.config.EnvConfigs; -import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; import io.airbyte.workers.process.ProcessFactory; import java.util.Map.Entry; @@ -32,11 +30,10 @@ void testMappings() { for (final Entry> entry : NormalizationRunnerFactory.NORMALIZATION_MAPPING.entrySet()) { assertEquals(entry.getValue().getValue(), ((DefaultNormalizationRunner) NormalizationRunnerFactory.create( - new WorkerConfigs(new EnvConfigs()), String.format("%s:0.1.0", entry.getKey()), processFactory, NORMALIZATION_VERSION)).getDestinationType()); } assertThrows(IllegalStateException.class, - () -> NormalizationRunnerFactory.create(new WorkerConfigs(new EnvConfigs()), "airbyte/destination-csv:0.1.0", processFactory, + () -> NormalizationRunnerFactory.create("airbyte/destination-csv:0.1.0", processFactory, NORMALIZATION_VERSION)); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java index 340049eb57bec0..ba817032096269 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java @@ -51,7 +51,7 @@ void testJqExists() throws IOException { LineGobbler.gobble(process.getInputStream(), out::append); LineGobbler.gobble(process.getErrorStream(), err::append); - WorkerUtils.gentleClose(new WorkerConfigs(new EnvConfigs()), process, 1, TimeUnit.MINUTES); + WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); assertEquals(0, process.exitValue(), String.format("Error while checking for jq. STDOUT: %s STDERR: %s Please make sure jq is installed (used by testImageExists)", out, err)); @@ -135,7 +135,7 @@ void testEnvMapSet() throws IOException, WorkerException, InterruptedException { LineGobbler.gobble(process.getInputStream(), out::append); LineGobbler.gobble(process.getErrorStream(), err::append); - WorkerUtils.gentleClose(new WorkerConfigs(new EnvConfigs()), process, 20, TimeUnit.SECONDS); + WorkerUtils.gentleClose(process, 20, TimeUnit.SECONDS); assertEquals(0, process.exitValue(), String.format("Process failed with stdout: %s and stderr: %s", out, err)); assertEquals("ENV_VAR_1=ENV_VALUE_1", out.toString(), String.format("Output did not contain the expected string. stdout: %s", out)); diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml index a4339fd541904f..9e40d680bdcb91 100644 --- a/charts/airbyte-worker/templates/deployment.yaml +++ b/charts/airbyte-worker/templates/deployment.yaml @@ -89,11 +89,6 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: WORKSPACE_ROOT - - name: WORKER_ENVIRONMENT - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: WORKER_ENVIRONMENT - name: LOCAL_ROOT valueFrom: configMapKeyRef: diff --git a/docker-compose.yaml b/docker-compose.yaml index 7e76bf56f43b2c..18cb71a5c31946 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -89,7 +89,6 @@ services: - TEMPORAL_HOST=${TEMPORAL_HOST} - TRACKING_STRATEGY=${TRACKING_STRATEGY} - WEBAPP_URL=${WEBAPP_URL} - - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT} - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT} - WORKSPACE_ROOT=${WORKSPACE_ROOT} - METRIC_CLIENT=${METRIC_CLIENT} diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 511e67e4b6e91c..7a2b424fc86175 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -65,11 +65,6 @@ spec: configMapKeyRef: name: airbyte-env key: WORKSPACE_ROOT - - name: WORKER_ENVIRONMENT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: WORKER_ENVIRONMENT - name: LOCAL_ROOT valueFrom: configMapKeyRef: From 7180ac3f688c48bbb0484a7e3c7e2b50a18ef17d Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 4 Oct 2022 14:43:56 -0700 Subject: [PATCH 26/32] Revert "Remove data field in the state aggregator if possible (#17538)" (#17583) This reverts commit f45275e78f5ec7dceb05fae0d0d02fffc22dcc10. --- .../SingleStateAggregator.java | 7 ---- .../StreamStateAggregator.java | 7 ---- .../state_aggregator/StateAggregatorTest.java | 33 +++++++------------ 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java index 0cfe422ea1f789..09106a08c1e243 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java @@ -24,13 +24,6 @@ public State getAggregated() { if (state.getType() == null || state.getType() == AirbyteStateType.LEGACY) { return new State().withState(state.getData()); } else { - /** - * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are - * running this code, we know that the platform has been upgraded and we can thus discard the legacy - * state. Keeping the legacy state is causing issue because of its size - * (https://github.com/airbytehq/oncall/issues/731) - */ - state.setData(null); return new State() .withState(Jsons.jsonNode(List.of(state))); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java index 4d3247b2549d01..d55563efe0ec61 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java @@ -17,13 +17,6 @@ class StreamStateAggregator implements StateAggregator { @Override public void ingest(final AirbyteStateMessage stateMessage) { - /** - * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are - * running this code, we know that the platform has been upgraded and we can thus discard the legacy - * state. Keeping the legacy state is causing issue because of its size - * (https://github.com/airbytehq/oncall/issues/731) - */ - stateMessage.setData(null); aggregatedState.put(stateMessage.getStream().getStreamDescriptor(), stateMessage); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java index 83761bf5b194fc..bd591be29b132b 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java @@ -92,16 +92,13 @@ void testGlobalState() { final AirbyteStateMessage state1 = getGlobalMessage(1); final AirbyteStateMessage state2 = getGlobalMessage(2); - final AirbyteStateMessage state1NoData = getGlobalMessage(1).withData(null); - final AirbyteStateMessage state2NoData = getGlobalMessage(2).withData(null); - - stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); + stateAggregator.ingest(state1); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1NoData)))); + .withState(Jsons.jsonNode(List.of(state1)))); - stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); + stateAggregator.ingest(state2); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2NoData)))); + .withState(Jsons.jsonNode(List.of(state2)))); } @Test @@ -129,23 +126,19 @@ void testStreamStateWithFeatureFlagOn() { final AirbyteStateMessage state2 = getStreamMessage("b", 2); final AirbyteStateMessage state3 = getStreamMessage("b", 3); - final AirbyteStateMessage state1NoData = getStreamMessage("a", 1).withData(null); - final AirbyteStateMessage state2NoData = getStreamMessage("b", 2).withData(null); - final AirbyteStateMessage state3NoData = getStreamMessage("b", 3).withData(null); - stateAggregator = new DefaultStateAggregator(USE_STREAM_CAPABLE_STATE); - stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); + stateAggregator.ingest(state1); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1NoData)))); + .withState(Jsons.jsonNode(List.of(state1)))); - stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); + stateAggregator.ingest(state2); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2NoData, state1NoData)))); + .withState(Jsons.jsonNode(List.of(state2, state1)))); - stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state3), AirbyteStateMessage.class)); + stateAggregator.ingest(state3); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state3NoData, state1NoData)))); + .withState(Jsons.jsonNode(List.of(state3, state1)))); } private AirbyteStateMessage getNullMessage(final int stateValue) { @@ -165,8 +158,7 @@ private AirbyteStateMessage getGlobalMessage(final int stateValue) { .withStreamDescriptor( new StreamDescriptor() .withName("test")) - .withStreamState(Jsons.jsonNode(stateValue))))) - .withData(Jsons.jsonNode("HelloWorld")); + .withStreamState(Jsons.jsonNode(stateValue))))); } private AirbyteStateMessage getStreamMessage(final String streamName, final int stateValue) { @@ -176,8 +168,7 @@ private AirbyteStateMessage getStreamMessage(final String streamName, final int .withStreamDescriptor( new StreamDescriptor() .withName(streamName)) - .withStreamState(Jsons.jsonNode(stateValue))) - .withData(Jsons.jsonNode("Hello")); + .withStreamState(Jsons.jsonNode(stateValue))); } private AirbyteStateMessage getEmptyMessage(final AirbyteStateType stateType) { From c886dc9da456c83d0fb63d4164a83dedf2ed2680 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 4 Oct 2022 15:01:14 -0700 Subject: [PATCH 27/32] Revert "Revert "Remove data field in the state aggregator if possible (#17538)" (#17583)" (#17584) This reverts commit 7180ac3f688c48bbb0484a7e3c7e2b50a18ef17d. --- .../SingleStateAggregator.java | 7 ++++ .../StreamStateAggregator.java | 7 ++++ .../state_aggregator/StateAggregatorTest.java | 33 ++++++++++++------- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java index 09106a08c1e243..0cfe422ea1f789 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java @@ -24,6 +24,13 @@ public State getAggregated() { if (state.getType() == null || state.getType() == AirbyteStateType.LEGACY) { return new State().withState(state.getData()); } else { + /** + * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are + * running this code, we know that the platform has been upgraded and we can thus discard the legacy + * state. Keeping the legacy state is causing issue because of its size + * (https://github.com/airbytehq/oncall/issues/731) + */ + state.setData(null); return new State() .withState(Jsons.jsonNode(List.of(state))); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java index d55563efe0ec61..4d3247b2549d01 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java @@ -17,6 +17,13 @@ class StreamStateAggregator implements StateAggregator { @Override public void ingest(final AirbyteStateMessage stateMessage) { + /** + * The destination emit a Legacy state in order to be retro-compatible with old platform. If we are + * running this code, we know that the platform has been upgraded and we can thus discard the legacy + * state. Keeping the legacy state is causing issue because of its size + * (https://github.com/airbytehq/oncall/issues/731) + */ + stateMessage.setData(null); aggregatedState.put(stateMessage.getStream().getStreamDescriptor(), stateMessage); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java index bd591be29b132b..83761bf5b194fc 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/state_aggregator/StateAggregatorTest.java @@ -92,13 +92,16 @@ void testGlobalState() { final AirbyteStateMessage state1 = getGlobalMessage(1); final AirbyteStateMessage state2 = getGlobalMessage(2); - stateAggregator.ingest(state1); + final AirbyteStateMessage state1NoData = getGlobalMessage(1).withData(null); + final AirbyteStateMessage state2NoData = getGlobalMessage(2).withData(null); + + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1)))); + .withState(Jsons.jsonNode(List.of(state1NoData)))); - stateAggregator.ingest(state2); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2)))); + .withState(Jsons.jsonNode(List.of(state2NoData)))); } @Test @@ -126,19 +129,23 @@ void testStreamStateWithFeatureFlagOn() { final AirbyteStateMessage state2 = getStreamMessage("b", 2); final AirbyteStateMessage state3 = getStreamMessage("b", 3); + final AirbyteStateMessage state1NoData = getStreamMessage("a", 1).withData(null); + final AirbyteStateMessage state2NoData = getStreamMessage("b", 2).withData(null); + final AirbyteStateMessage state3NoData = getStreamMessage("b", 3).withData(null); + stateAggregator = new DefaultStateAggregator(USE_STREAM_CAPABLE_STATE); - stateAggregator.ingest(state1); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state1), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state1)))); + .withState(Jsons.jsonNode(List.of(state1NoData)))); - stateAggregator.ingest(state2); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state2), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state2, state1)))); + .withState(Jsons.jsonNode(List.of(state2NoData, state1NoData)))); - stateAggregator.ingest(state3); + stateAggregator.ingest(Jsons.object(Jsons.jsonNode(state3), AirbyteStateMessage.class)); Assertions.assertThat(stateAggregator.getAggregated()).isEqualTo(new State() - .withState(Jsons.jsonNode(List.of(state3, state1)))); + .withState(Jsons.jsonNode(List.of(state3NoData, state1NoData)))); } private AirbyteStateMessage getNullMessage(final int stateValue) { @@ -158,7 +165,8 @@ private AirbyteStateMessage getGlobalMessage(final int stateValue) { .withStreamDescriptor( new StreamDescriptor() .withName("test")) - .withStreamState(Jsons.jsonNode(stateValue))))); + .withStreamState(Jsons.jsonNode(stateValue))))) + .withData(Jsons.jsonNode("HelloWorld")); } private AirbyteStateMessage getStreamMessage(final String streamName, final int stateValue) { @@ -168,7 +176,8 @@ private AirbyteStateMessage getStreamMessage(final String streamName, final int .withStreamDescriptor( new StreamDescriptor() .withName(streamName)) - .withStreamState(Jsons.jsonNode(stateValue))); + .withStreamState(Jsons.jsonNode(stateValue))) + .withData(Jsons.jsonNode("Hello")); } private AirbyteStateMessage getEmptyMessage(final AirbyteStateType stateType) { From 5ba79894f25c52c6b55cb4a4f2610379bb8defca Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Wed, 5 Oct 2022 09:20:08 +0300 Subject: [PATCH 28/32] Source Iterable: use optional end_date for SATs (#17573) * #17506 source iterable: use optional end_date for SATs * source iterable: upd changelog * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-iterable/Dockerfile | 2 +- .../source_iterable/slice_generators.py | 13 ++-- .../source-iterable/source_iterable/source.py | 77 ++++++++++--------- .../source_iterable/streams.py | 15 ++-- docs/integrations/sources/iterable.md | 1 + 7 files changed, 58 insertions(+), 54 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d9136a0c2c55aa..fa5c6149df0f00 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -504,7 +504,7 @@ - name: Iterable sourceDefinitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 dockerRepository: airbyte/source-iterable - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/sources/iterable icon: iterable.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e048fe4975f30d..9db789112d6162 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5002,7 +5002,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-iterable:0.1.17" +- dockerImage: "airbyte/source-iterable:0.1.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/iterable" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index 9aa55203fc034e..3a593b4f29409f 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py b/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py index 1eca994383c5a1..c1a3f855fcb099 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py @@ -4,7 +4,7 @@ import math from dataclasses import dataclass -from typing import Iterable, List, Tuple +from typing import Iterable, List, Optional, Tuple import pendulum from pendulum.datetime import DateTime, Period @@ -24,9 +24,9 @@ class SliceGenerator: _start_date: DateTime = None _end_data: DateTime = None - def __init__(self, start_date: DateTime): + def __init__(self, start_date: DateTime, end_date: Optional[DateTime] = None): self._start_date = start_date - self._end_date = pendulum.now("UTC") + self._end_date = end_date or pendulum.now("UTC") def __iter__(self): return self @@ -41,8 +41,8 @@ class RangeSliceGenerator(SliceGenerator): RANGE_LENGTH_DAYS: int = 90 _slices: List[StreamSlice] = [] - def __init__(self, start_date: DateTime): - super().__init__(start_date) + def __init__(self, start_date: DateTime, end_date: Optional[DateTime] = None): + super().__init__(start_date, end_date) self._slices = [ StreamSlice(start_date=start, end_date=end) for start, end in self.make_datetime_ranges(self._start_date, self._end_date, self.RANGE_LENGTH_DAYS) @@ -120,9 +120,6 @@ class AdjustableSliceGenerator(SliceGenerator): # Default is True so for first slice it would length would be INITIAL_RANGE_DAYS (30 days) _range_adjusted = True - def __init__(self, start_date: DateTime): - super().__init__(start_date) - def adjust_range(self, previous_request_time: Period): """ Calculate next slice length in days based on previous slice length and diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 9789e9f5727f40..2cfe9358bfd1ca 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -76,49 +76,52 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="") + # end date is provided for integration tests only + start_date, end_date = config["start_date"], config.get("end_date") + date_range = {"start_date": start_date, "end_date": end_date} return [ Campaigns(authenticator=authenticator), - CampaignsMetrics(authenticator=authenticator, start_date=config["start_date"]), + CampaignsMetrics(authenticator=authenticator, **date_range), Channels(authenticator=authenticator), - EmailBounce(authenticator=authenticator, start_date=config["start_date"]), - EmailClick(authenticator=authenticator, start_date=config["start_date"]), - EmailComplaint(authenticator=authenticator, start_date=config["start_date"]), - EmailOpen(authenticator=authenticator, start_date=config["start_date"]), - EmailSend(authenticator=authenticator, start_date=config["start_date"]), - EmailSendSkip(authenticator=authenticator, start_date=config["start_date"]), - EmailSubscribe(authenticator=authenticator, start_date=config["start_date"]), - EmailUnsubscribe(authenticator=authenticator, start_date=config["start_date"]), - PushSend(authenticator=authenticator, start_date=config["start_date"]), - PushSendSkip(authenticator=authenticator, start_date=config["start_date"]), - PushOpen(authenticator=authenticator, start_date=config["start_date"]), - PushUninstall(authenticator=authenticator, start_date=config["start_date"]), - PushBounce(authenticator=authenticator, start_date=config["start_date"]), - WebPushSend(authenticator=authenticator, start_date=config["start_date"]), - WebPushClick(authenticator=authenticator, start_date=config["start_date"]), - WebPushSendSkip(authenticator=authenticator, start_date=config["start_date"]), - InAppSend(authenticator=authenticator, start_date=config["start_date"]), - InAppOpen(authenticator=authenticator, start_date=config["start_date"]), - InAppClick(authenticator=authenticator, start_date=config["start_date"]), - InAppClose(authenticator=authenticator, start_date=config["start_date"]), - InAppDelete(authenticator=authenticator, start_date=config["start_date"]), - InAppDelivery(authenticator=authenticator, start_date=config["start_date"]), - InAppSendSkip(authenticator=authenticator, start_date=config["start_date"]), - InboxSession(authenticator=authenticator, start_date=config["start_date"]), - InboxMessageImpression(authenticator=authenticator, start_date=config["start_date"]), - SmsSend(authenticator=authenticator, start_date=config["start_date"]), - SmsBounce(authenticator=authenticator, start_date=config["start_date"]), - SmsClick(authenticator=authenticator, start_date=config["start_date"]), - SmsReceived(authenticator=authenticator, start_date=config["start_date"]), - SmsSendSkip(authenticator=authenticator, start_date=config["start_date"]), - SmsUsageInfo(authenticator=authenticator, start_date=config["start_date"]), - Purchase(authenticator=authenticator, start_date=config["start_date"]), - CustomEvent(authenticator=authenticator, start_date=config["start_date"]), - HostedUnsubscribeClick(authenticator=authenticator, start_date=config["start_date"]), + EmailBounce(authenticator=authenticator, **date_range), + EmailClick(authenticator=authenticator, **date_range), + EmailComplaint(authenticator=authenticator, **date_range), + EmailOpen(authenticator=authenticator, **date_range), + EmailSend(authenticator=authenticator, **date_range), + EmailSendSkip(authenticator=authenticator, **date_range), + EmailSubscribe(authenticator=authenticator, **date_range), + EmailUnsubscribe(authenticator=authenticator, **date_range), + PushSend(authenticator=authenticator, **date_range), + PushSendSkip(authenticator=authenticator, **date_range), + PushOpen(authenticator=authenticator, **date_range), + PushUninstall(authenticator=authenticator, **date_range), + PushBounce(authenticator=authenticator, **date_range), + WebPushSend(authenticator=authenticator, **date_range), + WebPushClick(authenticator=authenticator, **date_range), + WebPushSendSkip(authenticator=authenticator, **date_range), + InAppSend(authenticator=authenticator, **date_range), + InAppOpen(authenticator=authenticator, **date_range), + InAppClick(authenticator=authenticator, **date_range), + InAppClose(authenticator=authenticator, **date_range), + InAppDelete(authenticator=authenticator, **date_range), + InAppDelivery(authenticator=authenticator, **date_range), + InAppSendSkip(authenticator=authenticator, **date_range), + InboxSession(authenticator=authenticator, **date_range), + InboxMessageImpression(authenticator=authenticator, **date_range), + SmsSend(authenticator=authenticator, **date_range), + SmsBounce(authenticator=authenticator, **date_range), + SmsClick(authenticator=authenticator, **date_range), + SmsReceived(authenticator=authenticator, **date_range), + SmsSendSkip(authenticator=authenticator, **date_range), + SmsUsageInfo(authenticator=authenticator, **date_range), + Purchase(authenticator=authenticator, **date_range), + CustomEvent(authenticator=authenticator, **date_range), + HostedUnsubscribeClick(authenticator=authenticator, **date_range), Events(authenticator=authenticator), Lists(authenticator=authenticator), ListUsers(authenticator=authenticator), MessageTypes(authenticator=authenticator), Metadata(authenticator=authenticator), - Templates(authenticator=authenticator, start_date=config["start_date"]), - Users(authenticator=authenticator, start_date=config["start_date"]), + Templates(authenticator=authenticator, **date_range), + Users(authenticator=authenticator, **date_range), ] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index 244f080023d09a..c3613d272d8386 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -75,9 +75,10 @@ class IterableExportStream(IterableStream, ABC): cursor_field = "createdAt" primary_key = None - def __init__(self, start_date=None, **kwargs): + def __init__(self, start_date=None, end_date=None, **kwargs): super().__init__(**kwargs) self._start_date = pendulum.parse(start_date) + self._end_date = end_date and pendulum.parse(end_date) self.stream_params = {"dataTypeName": self.data_field} def path(self, **kwargs) -> str: @@ -185,7 +186,7 @@ def stream_slices( ) -> Iterable[Optional[StreamSlice]]: start_datetime = self.get_start_date(stream_state) - return [StreamSlice(start_datetime, pendulum.now("UTC"))] + return [StreamSlice(start_datetime, self._end_date or pendulum.now("UTC"))] class IterableExportStreamRanged(IterableExportStream, ABC): @@ -204,7 +205,7 @@ def stream_slices( start_datetime = self.get_start_date(stream_state) - return RangeSliceGenerator(start_datetime) + return RangeSliceGenerator(start_datetime, self._end_date) class IterableExportStreamAdjustableRange(IterableExportStream, ABC): @@ -238,7 +239,7 @@ def stream_slices( ) -> Iterable[Optional[StreamSlice]]: start_datetime = self.get_start_date(stream_state) - self._adjustable_generator = AdjustableSliceGenerator(start_datetime) + self._adjustable_generator = AdjustableSliceGenerator(start_datetime, self._end_date) return self._adjustable_generator def read_records( @@ -325,12 +326,13 @@ class CampaignsMetrics(IterableStream): primary_key = None data_field = None - def __init__(self, start_date: str, **kwargs): + def __init__(self, start_date: str, end_date: Optional[str] = None, **kwargs): """ https://api.iterable.com/api/docs#campaigns_metrics """ super().__init__(**kwargs) self.start_date = start_date + self.end_date = end_date def path(self, **kwargs) -> str: return "campaigns/metrics" @@ -339,7 +341,8 @@ def request_params(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwa params = super().request_params(**kwargs) params["campaignId"] = stream_slice.get("campaign_ids") params["startDateTime"] = self.start_date - + if self.end_date: + params["endDateTime"] = self.end_date return params def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index ff08803e187d57..c278e448018d4e 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -92,6 +92,7 @@ The Iterable source connector supports the following [sync modes](https://docs.a | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.18 | 2022-10-04 | [17573](https://github.com/airbytehq/airbyte/pull/17573) | Limit time range for SATs | | 0.1.17 | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams | | 0.1.16 | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header | | 0.1.15 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | From cb0cb925ff71ba59b24414cb2e0bb696b59436e0 Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:21:17 +0200 Subject: [PATCH 29/32] =?UTF-8?q?=F0=9F=AA=9F=20=20=F0=9F=94=A7=20Cleanup?= =?UTF-8?q?=20UI=20component=20imports=20(#17350)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Clean up UI comonent imports * Clean up DropDown option component import * Clean up Dropdown SingleValue imports * Remove default exports from ui and other minor cleanup * Update snapshots --- .../components/EditorHeader.tsx | 2 +- .../components/EditorRow.tsx | 2 +- .../ConnectorBlocks/TableItemTitle.tsx | 7 ++-- .../CreateConnection/CreateConnection.tsx | 3 +- .../EntityTable/components/StatusCell.tsx | 3 +- .../JobItem/components/DownloadButton.tsx | 2 +- .../LabeledRadioButton/LabeledRadioButton.tsx | 2 +- .../LabeledSwitch/LabeledSwitch.tsx | 3 +- airbyte-webapp/src/components/index.tsx | 2 - .../src/components/ui/DropDown/DropDown.tsx | 10 ++--- .../components/{Text.tsx => DropDownText.tsx} | 8 ++-- .../ui/DropDown/components/Menu.tsx | 6 +-- .../ui/DropDown/components/Option.tsx | 16 ++++---- .../ui/DropDown/components/SingleValue.tsx | 40 +++++++++---------- .../src/components/ui/DropDown/index.tsx | 6 +-- .../components/ui/ImageBlock/ImageBlock.tsx | 2 - .../src/components/ui/Input/Input.tsx | 2 - .../src/components/ui/Input/index.stories.tsx | 2 +- .../src/components/ui/Input/index.tsx | 4 -- .../components/ui/Multiselect/Multiselect.tsx | 9 ++--- .../src/components/ui/Popout/Popout.tsx | 3 +- .../components/ui/RadioButton/RadioButton.tsx | 4 +- .../src/components/ui/RadioButton/index.tsx | 5 +-- .../src/components/ui/SideMenu/MenuItem.tsx | 4 +- .../src/components/ui/SideMenu/SideMenu.tsx | 2 +- .../src/components/ui/StepsMenu/Step.tsx | 4 +- .../src/components/ui/StepsMenu/StepsMenu.tsx | 2 +- .../src/components/ui/TagInput/TagInput.tsx | 2 +- .../src/components/ui/TagInput/TagItem.tsx | 4 +- .../src/components/ui/TextArea/index.tsx | 4 -- .../src/components/ui/Toast/ErrorSign.tsx | 4 +- ...etonCard.module.scss => Toast.module.scss} | 0 .../src/components/ui/Toast/Toast.tsx | 6 +-- airbyte-webapp/src/components/ui/index.tsx | 11 ----- .../ConfirmPasswordResetPage.tsx | 3 +- .../cloud/views/auth/LoginPage/LoginPage.tsx | 3 +- .../ResetPasswordPage/ResetPasswordPage.tsx | 3 +- .../auth/SignupPage/components/SignupForm.tsx | 3 +- .../views/auth/components/Header/Header.tsx | 5 ++- .../components/RemainingCredits.tsx | 2 +- .../AccountSettingsView.tsx | 2 +- .../components/EmailSection/EmailSection.tsx | 2 +- .../components/NameSection/NameSection.tsx | 2 +- .../PasswordSection/PasswordSection.tsx | 2 +- .../InviteUsersModal/InviteUsersModal.tsx | 4 +- .../WorkspacePopout/WorkspacePopout.tsx | 6 +-- .../components/CreateWorkspaceForm.tsx | 3 +- .../components/WorkspacesControl.tsx | 3 +- .../AllConnectionsPage/AllConnectionsPage.tsx | 3 +- .../ConnectionItemPage/ConnectionName.tsx | 2 +- .../ConnectionItemPage/EnabledControl.tsx | 2 +- .../CreationFormPage/ExistingEntityForm.tsx | 4 +- .../AllDestinationsPage.tsx | 3 +- .../DestinationItemPage.tsx | 5 ++- .../pages/OnboardingPage/OnboardingPage.tsx | 2 +- .../components/ProgressBlock.tsx | 6 +-- .../OnboardingPage/components/WelcomeStep.tsx | 2 +- .../components/LogsContent.tsx | 2 +- .../components/CreateConnector.tsx | 2 +- .../components/CreateConnectorModal.tsx | 3 +- .../components/UpgradeAllButton.tsx | 2 +- .../ConnectorsPage/components/VersionCell.tsx | 3 +- .../components/WebHookForm.tsx | 4 +- .../pages/AllSourcesPage/AllSourcesPage.tsx | 3 +- .../pages/SourceItemPage/SourceItemPage.tsx | 4 +- .../CatalogDiffModal/CatalogDiffModal.tsx | 2 +- .../Connection/CatalogTree/CatalogSection.tsx | 4 +- .../views/Connection/CatalogTree/FieldRow.tsx | 6 ++- .../Connection/CatalogTree/StreamHeader.tsx | 7 +++- .../CatalogTree/components/BulkHeader.tsx | 4 +- .../components/SyncSettingsDropdown.tsx | 20 ++++++---- .../components/CreateControls.tsx | 3 +- .../components/EditControls.tsx | 2 +- .../components/NamespaceDefinitionField.tsx | 3 +- .../components/ScheduleField.tsx | 17 ++++---- .../ConnectionForm/components/Search.tsx | 2 +- .../ConnectionForm/components/Section.tsx | 2 +- .../Connection/ConnectionForm/formConfig.tsx | 4 +- .../Connector/ConnectorCard/ConnectorCard.tsx | 2 +- .../components/ConnectorForm.tsx | 5 ++- .../Controls/ConnectorNameControl.tsx | 2 +- .../Controls/ConnectorServiceTypeControl.tsx | 21 +++++----- .../ServiceForm/components/CreateControls.tsx | 2 +- .../ServiceForm/components/EditControls.tsx | 2 +- .../FrequentlyUsedDestinations.test.tsx.snap | 14 +++---- .../Property/ConfirmationControl.tsx | 2 +- .../components/Property/Control.tsx | 6 ++- .../Property/PropertyLabel.stories.tsx | 3 +- .../components/Sections/ConditionSection.tsx | 5 +-- .../components/Sections/auth/AuthButton.tsx | 2 +- .../StartWithDestination.tsx | 3 +- .../StartWithDestination.test.tsx.snap | 4 +- .../components/TestingConnectionSpinner.tsx | 2 +- .../PreferencesForm/PreferencesForm.tsx | 2 +- .../components/EditControls.tsx | 2 +- 95 files changed, 217 insertions(+), 219 deletions(-) rename airbyte-webapp/src/components/ui/DropDown/components/{Text.tsx => DropDownText.tsx} (83%) rename airbyte-webapp/src/components/ui/Toast/{SingletonCard.module.scss => Toast.module.scss} (100%) delete mode 100644 airbyte-webapp/src/components/ui/index.tsx diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx index e8f6760144aab7..467e5655fd6a79 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { ConnectionFormMode } from "views/Connection/ConnectionForm/ConnectionForm"; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx index 7ce55c8de17802..2184263b3383f2 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx @@ -1,9 +1,9 @@ import React from "react"; import { useIntl } from "react-intl"; -import { Button } from "components"; import { CrossIcon } from "components/icons/CrossIcon"; import { PencilIcon } from "components/icons/PencilIcon"; +import { Button } from "components/ui/Button"; import { Tooltip } from "components/ui/Tooltip"; import styles from "./EditorRow.module.scss"; diff --git a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx index 7c6bec992db95c..f51a44621c4708 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx @@ -1,8 +1,9 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { Button, DropDownRow } from "components"; import { ReleaseStageBadge } from "components/ReleaseStageBadge"; +import { Button } from "components/ui/Button"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; import { Popout } from "components/ui/Popout"; import { Text } from "components/ui/Text"; @@ -13,8 +14,8 @@ import styles from "./TableItemTitle.module.scss"; interface TableItemTitleProps { type: "source" | "destination"; - dropDownData: DropDownRow.IDataItem[]; - onSelect: (item: DropDownRow.IDataItem) => void; + dropDownData: DropDownOptionDataItem[]; + onSelect: (item: DropDownOptionDataItem) => void; entity: string; entityName: string; entityIcon?: React.ReactNode; diff --git a/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx b/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx index d84727c643bdaa..4f21e6414d0fc9 100644 --- a/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx +++ b/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx @@ -4,9 +4,10 @@ import React, { Suspense, useCallback, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button, Card } from "components"; import { JobItem } from "components/JobItem/JobItem"; import LoadingSchema from "components/LoadingSchema"; +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; import { LogsRequestError } from "core/request/LogsRequestError"; import { ConnectionFormServiceProvider } from "hooks/services/ConnectionForm/ConnectionFormService"; diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx index 2eb2072fbc2f33..b106e7bc2d4196 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx @@ -3,7 +3,8 @@ import { FormattedMessage } from "react-intl"; import { useAsyncFn } from "react-use"; import styled from "styled-components"; -import { Button, Switch } from "components"; +import { Button } from "components/ui/Button"; +import { Switch } from "components/ui/Switch"; import { useEnableConnection } from "hooks/services/useConnectionHook"; diff --git a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx index d49df3f3508bcf..892e0464637aa9 100644 --- a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx +++ b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx @@ -3,7 +3,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { useIntl } from "react-intl"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { JobDebugInfoRead } from "core/request/AirbyteClient"; import { useCurrentWorkspaceId, useGetWorkspace } from "services/workspaces/WorkspacesService"; diff --git a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx index 7192223b2b31e3..7026324a0f496a 100644 --- a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx +++ b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx @@ -1,7 +1,7 @@ import React from "react"; import styled from "styled-components"; -import { RadioButton } from "components"; +import { RadioButton } from "components/ui/RadioButton"; type IProps = { message?: React.ReactNode; diff --git a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx index 31b95f86757efb..48f2e9880af17f 100644 --- a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx +++ b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx @@ -1,7 +1,8 @@ import classNames from "classnames"; import React from "react"; -import { CheckBox, Switch } from "components/ui"; +import { CheckBox } from "components/ui/CheckBox"; +import { Switch } from "components/ui/Switch"; import styles from "./LabeledSwitch.module.scss"; diff --git a/airbyte-webapp/src/components/index.tsx b/airbyte-webapp/src/components/index.tsx index 550e525f17da15..54763971f8c3cf 100644 --- a/airbyte-webapp/src/components/index.tsx +++ b/airbyte-webapp/src/components/index.tsx @@ -1,5 +1,3 @@ -export * from "./ui"; - export * from "./ArrayOfObjectsEditor"; export * from "./DefaultLogoCatalog"; export * from "./Label"; diff --git a/airbyte-webapp/src/components/ui/DropDown/DropDown.tsx b/airbyte-webapp/src/components/ui/DropDown/DropDown.tsx index c430e4e478f5a9..9fd38556a6cf77 100644 --- a/airbyte-webapp/src/components/ui/DropDown/DropDown.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/DropDown.tsx @@ -6,9 +6,9 @@ import Select from "react-select/dist/declarations/src/Select"; import { equal, naturalComparatorBy } from "utils/objects"; import { DropdownIndicator } from "./components/DropdownIndicator"; -import Menu from "./components/Menu"; -import Option, { IDataItem } from "./components/Option"; -import SingleValue from "./components/SingleValue"; +import { Menu } from "./components/Menu"; +import { DropDownOption, DropDownOptionDataItem } from "./components/Option"; +import { SingleValue } from "./components/SingleValue"; import { CustomSelect } from "./CustomSelect"; import { SelectContainer } from "./SelectContainer"; @@ -36,7 +36,7 @@ function DropDownInner( DropdownIndicator, SelectContainer, Menu, - Option, + Option: DropDownOption, SingleValue, IndicatorSeparator: null, ClearIndicator: null, @@ -84,6 +84,6 @@ function DropDownInner( ); } -export const defaultDataItemSort = naturalComparatorBy((dataItem) => dataItem.label || ""); +export const defaultDataItemSort = naturalComparatorBy((dataItem) => dataItem.label || ""); export const DropDown = React.forwardRef(DropDownInner); diff --git a/airbyte-webapp/src/components/ui/DropDown/components/Text.tsx b/airbyte-webapp/src/components/ui/DropDown/components/DropDownText.tsx similarity index 83% rename from airbyte-webapp/src/components/ui/DropDown/components/Text.tsx rename to airbyte-webapp/src/components/ui/DropDown/components/DropDownText.tsx index e642275ce2a219..2db3be0bc465ae 100644 --- a/airbyte-webapp/src/components/ui/DropDown/components/Text.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/components/DropDownText.tsx @@ -1,13 +1,13 @@ import styled from "styled-components"; import { Theme } from "theme"; -interface IProps { +interface DropDownTextProps { primary?: boolean; secondary?: boolean; fullText?: boolean; } -const setColor = (props: IProps & { theme: Theme }) => { +const setColor = (props: DropDownTextProps & { theme: Theme }) => { if (props.primary) { return props.theme.primaryColor; } @@ -18,7 +18,7 @@ const setColor = (props: IProps & { theme: Theme }) => { return "inherit"; }; -const Text = styled.div` +export const DropDownText = styled.div` white-space: ${({ fullText }) => (fullText ? "normal" : "nowrap")}; overflow: ${({ fullText }) => (fullText ? "inherit" : "hidden")}; text-overflow: ${({ fullText }) => (fullText ? "inherit" : "ellipsis")}; @@ -33,5 +33,3 @@ const Text = styled.div` color: ${({ theme }) => theme.primaryColor}; } `; - -export default Text; diff --git a/airbyte-webapp/src/components/ui/DropDown/components/Menu.tsx b/airbyte-webapp/src/components/ui/DropDown/components/Menu.tsx index 723d838a6356e5..40adbea106d498 100644 --- a/airbyte-webapp/src/components/ui/DropDown/components/Menu.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/components/Menu.tsx @@ -1,9 +1,9 @@ import { components, MenuProps } from "react-select"; import styled from "styled-components"; -import { IDataItem } from "./Option"; +import { DropDownOptionDataItem } from "./Option"; -const Menu = styled(components.Menu)>` +export const Menu = styled(components.Menu)>` background: ${({ theme }) => theme.textColor}; &.react-select__menu { @@ -26,5 +26,3 @@ const Menu = styled(components.Menu)>` } } `; - -export default Menu; diff --git a/airbyte-webapp/src/components/ui/DropDown/components/Option.tsx b/airbyte-webapp/src/components/ui/DropDown/components/Option.tsx index 3ae388c635bd8d..5de3503e106c37 100644 --- a/airbyte-webapp/src/components/ui/DropDown/components/Option.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/components/Option.tsx @@ -5,13 +5,13 @@ import styled from "styled-components"; import { CheckBox } from "components/ui/CheckBox"; import { OptionType } from "../DropDown"; -import Text from "./Text"; +import { DropDownText } from "./DropDownText"; -export type IProps = { - data: { disabled: boolean; index: number; fullText?: boolean } & IDataItem; +export type DropDownOptionProps = { + data: { disabled: boolean; index: number; fullText?: boolean } & DropDownOptionDataItem; } & OptionProps; -export interface IDataItem { +export interface DropDownOptionDataItem { label?: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any value?: any; @@ -45,7 +45,7 @@ export const OptionView = styled.div<{ } `; -const Option: React.FC = (props) => { +export const DropDownOption: React.FC = (props) => { const dataTestId = props.data.testId ? props.data.testId : !["object", "array"].includes(typeof props.data.label) @@ -59,18 +59,16 @@ const Option: React.FC = (props) => { isSelected={props.isSelected && !props.isMulti} isDisabled={props.isDisabled} > - + {props.isMulti && ( <> props.selectOption(props.data)} />{" "} )} {props.label} - + {props.data.img || null} ); }; - -export default Option; diff --git a/airbyte-webapp/src/components/ui/DropDown/components/SingleValue.tsx b/airbyte-webapp/src/components/ui/DropDown/components/SingleValue.tsx index 9499caa1fa93dc..d3edb8491d6105 100644 --- a/airbyte-webapp/src/components/ui/DropDown/components/SingleValue.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/components/SingleValue.tsx @@ -1,35 +1,35 @@ import React from "react"; -import { components, SingleValueProps } from "react-select"; +import { components, SingleValueProps as ReactSelectSingleValueProps } from "react-select"; import styled from "styled-components"; -import { IDataItem } from "./Option"; -import Text from "./Text"; +import { DropDownText } from "./DropDownText"; +import { DropDownOptionDataItem } from "./Option"; -export type IProps = { - data?: IDataItem; -} & SingleValueProps; +export type SingleValueProps = { + data?: DropDownOptionDataItem; +} & ReactSelectSingleValueProps; -export const ItemView = styled.div` +export const SingleValueView = styled.div` display: flex; flex-direction: row; justify-content: left; align-items: center; `; -export const Icon = styled.div` +export const SingleValueIcon = styled.div` margin-right: 6px; display: inline-block; `; -const SingleValue = (props: React.PropsWithChildren>) => { - return ( - - {props.data.img ? {props.data.img} : null} - - {props.children} - - - ); -}; - -export default React.memo(SingleValue); +export const SingleValue = React.memo( + (props: React.PropsWithChildren>) => { + return ( + + {props.data.img ? {props.data.img} : null} + + {props.children} + + + ); + } +); diff --git a/airbyte-webapp/src/components/ui/DropDown/index.tsx b/airbyte-webapp/src/components/ui/DropDown/index.tsx index d6d348bca7a1aa..141ffd026842ae 100644 --- a/airbyte-webapp/src/components/ui/DropDown/index.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/index.tsx @@ -1,4 +1,4 @@ -import * as DropDownRow from "./components/Option"; - +export * from "./components/Option"; +export * from "./components/SingleValue"; +export * from "./components/DropDownText"; export * from "./DropDown"; -export { DropDownRow }; diff --git a/airbyte-webapp/src/components/ui/ImageBlock/ImageBlock.tsx b/airbyte-webapp/src/components/ui/ImageBlock/ImageBlock.tsx index a239e42d715c27..6c0b75deae3597 100644 --- a/airbyte-webapp/src/components/ui/ImageBlock/ImageBlock.tsx +++ b/airbyte-webapp/src/components/ui/ImageBlock/ImageBlock.tsx @@ -34,5 +34,3 @@ export const ImageBlock: React.FC = ({ img, num, small, color, ); }; - -export default ImageBlock; diff --git a/airbyte-webapp/src/components/ui/Input/Input.tsx b/airbyte-webapp/src/components/ui/Input/Input.tsx index dcc8d2aec09bda..98f458e48bb75f 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.tsx +++ b/airbyte-webapp/src/components/ui/Input/Input.tsx @@ -115,5 +115,3 @@ export const Input: React.FC = ({ light, error, ...props }) => { ); }; - -export default Input; diff --git a/airbyte-webapp/src/components/ui/Input/index.stories.tsx b/airbyte-webapp/src/components/ui/Input/index.stories.tsx index cedfa6988a8fe9..0cc413d0d212d3 100644 --- a/airbyte-webapp/src/components/ui/Input/index.stories.tsx +++ b/airbyte-webapp/src/components/ui/Input/index.stories.tsx @@ -1,6 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; -import Input from "./Input"; +import { Input } from "./Input"; export default { title: "Ui/Input", diff --git a/airbyte-webapp/src/components/ui/Input/index.tsx b/airbyte-webapp/src/components/ui/Input/index.tsx index 4dfbc078c8ebbc..be66d7661c59f1 100644 --- a/airbyte-webapp/src/components/ui/Input/index.tsx +++ b/airbyte-webapp/src/components/ui/Input/index.tsx @@ -1,5 +1 @@ -import Input from "./Input"; - export * from "./Input"; - -export default Input; diff --git a/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx index 6cbe8cfb90219a..ffb99825f53a60 100644 --- a/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx +++ b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx @@ -3,13 +3,13 @@ import { MultiselectProps as WidgetMultiselectProps } from "react-widgets/lib/Mu import styled from "styled-components"; import "react-widgets/dist/css/react-widgets.css"; -type MultiselectProps = { +export interface MultiselectProps extends WidgetMultiselectProps { disabled?: boolean; error?: boolean; name?: string; -} & WidgetMultiselectProps; +} -const Multiselect = styled(ReactMultiselect)` +export const Multiselect = styled(ReactMultiselect)` box-shadow: none; padding: 0; margin: 0; @@ -129,6 +129,3 @@ const Multiselect = styled(ReactMultiselect)` } } `; - -export { Multiselect }; -export type { MultiselectProps }; diff --git a/airbyte-webapp/src/components/ui/Popout/Popout.tsx b/airbyte-webapp/src/components/ui/Popout/Popout.tsx index 65ece47a34bf94..0a483ea73c0568 100644 --- a/airbyte-webapp/src/components/ui/Popout/Popout.tsx +++ b/airbyte-webapp/src/components/ui/Popout/Popout.tsx @@ -3,8 +3,7 @@ import { ActionMeta, ControlProps, StylesConfig } from "react-select"; import { useToggle } from "react-use"; import styled from "styled-components"; -import { DropDown } from "components"; -import { DropdownProps } from "components/ui/DropDown"; +import { DropDown, DropdownProps } from "components/ui/DropDown"; const OutsideClickListener = styled.div` bottom: 0; diff --git a/airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx b/airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx index 4a9e883af39818..f56b52b195cd18 100644 --- a/airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx +++ b/airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx @@ -29,7 +29,7 @@ const RadioButtonContainer = styled.label<{ checked?: boolean; disabled?: boolea cursor: pointer; `; -const RadioButton: React.FC> = (props) => { +export const RadioButton: React.FC> = (props) => { return ( > = (prop ); }; - -export default RadioButton; diff --git a/airbyte-webapp/src/components/ui/RadioButton/index.tsx b/airbyte-webapp/src/components/ui/RadioButton/index.tsx index afc61c972c0e40..557d15219383a3 100644 --- a/airbyte-webapp/src/components/ui/RadioButton/index.tsx +++ b/airbyte-webapp/src/components/ui/RadioButton/index.tsx @@ -1,4 +1 @@ -import RadioButton from "./RadioButton"; - -export default RadioButton; -export { RadioButton }; +export * from "./RadioButton"; diff --git a/airbyte-webapp/src/components/ui/SideMenu/MenuItem.tsx b/airbyte-webapp/src/components/ui/SideMenu/MenuItem.tsx index bcc3088dfe8f95..5a3a4829f89108 100644 --- a/airbyte-webapp/src/components/ui/SideMenu/MenuItem.tsx +++ b/airbyte-webapp/src/components/ui/SideMenu/MenuItem.tsx @@ -38,7 +38,7 @@ const Counter = styled.div` margin-left: 5px; `; -const MenuItem: React.FC = ({ count, isActive, name, id, onClick }) => { +export const MenuItem: React.FC = ({ count, isActive, name, id, onClick }) => { return ( {name} @@ -46,5 +46,3 @@ const MenuItem: React.FC = ({ count, isActive, name, id, onClick }) => { ); }; - -export default MenuItem; diff --git a/airbyte-webapp/src/components/ui/SideMenu/SideMenu.tsx b/airbyte-webapp/src/components/ui/SideMenu/SideMenu.tsx index 2d51d2bae50339..3a1a22118a9323 100644 --- a/airbyte-webapp/src/components/ui/SideMenu/SideMenu.tsx +++ b/airbyte-webapp/src/components/ui/SideMenu/SideMenu.tsx @@ -1,7 +1,7 @@ import React from "react"; import styled from "styled-components"; -import MenuItem from "./MenuItem"; +import { MenuItem } from "./MenuItem"; export interface SideMenuItem { path: string; diff --git a/airbyte-webapp/src/components/ui/StepsMenu/Step.tsx b/airbyte-webapp/src/components/ui/StepsMenu/Step.tsx index 1c7db0e75406ba..ebba842db54d9a 100644 --- a/airbyte-webapp/src/components/ui/StepsMenu/Step.tsx +++ b/airbyte-webapp/src/components/ui/StepsMenu/Step.tsx @@ -57,7 +57,7 @@ const Num = styled.div<{ isActive?: boolean }>` box-shadow: 0 1px 2px 0 ${({ theme }) => theme.shadowColor}; `; -const Step: React.FC = ({ name, id, isActive, onClick, num, lightMode, status, isPartialSuccess }) => { +export const Step: React.FC = ({ name, id, isActive, onClick, num, lightMode, status, isPartialSuccess }) => { const onItemClickItem = () => { if (onClick) { onClick(id); @@ -83,5 +83,3 @@ const Step: React.FC = ({ name, id, isActive, onClick, num, lightMode, s ); }; - -export default Step; diff --git a/airbyte-webapp/src/components/ui/StepsMenu/StepsMenu.tsx b/airbyte-webapp/src/components/ui/StepsMenu/StepsMenu.tsx index 32cef9db1c5093..5cba7135e7b51e 100644 --- a/airbyte-webapp/src/components/ui/StepsMenu/StepsMenu.tsx +++ b/airbyte-webapp/src/components/ui/StepsMenu/StepsMenu.tsx @@ -1,7 +1,7 @@ import React from "react"; import styled from "styled-components"; -import Step from "./Step"; +import { Step } from "./Step"; export interface StepMenuItem { id: string; diff --git a/airbyte-webapp/src/components/ui/TagInput/TagInput.tsx b/airbyte-webapp/src/components/ui/TagInput/TagInput.tsx index 9921d41df57880..8f3f6123ffd3b1 100644 --- a/airbyte-webapp/src/components/ui/TagInput/TagInput.tsx +++ b/airbyte-webapp/src/components/ui/TagInput/TagInput.tsx @@ -1,7 +1,7 @@ import React, { useRef, useState } from "react"; import styled from "styled-components"; -import TagItem, { IItemProps } from "./TagItem"; +import { TagItem, IItemProps } from "./TagItem"; const MainContainer = styled.div<{ error?: boolean; disabled?: boolean }>` width: 100%; diff --git a/airbyte-webapp/src/components/ui/TagInput/TagItem.tsx b/airbyte-webapp/src/components/ui/TagInput/TagItem.tsx index 8580d705f8cfb2..1cf6f6aeb5d938 100644 --- a/airbyte-webapp/src/components/ui/TagInput/TagItem.tsx +++ b/airbyte-webapp/src/components/ui/TagInput/TagItem.tsx @@ -50,7 +50,7 @@ export interface IItemProps { id: string; } -const TagItem: React.FC = ({ item, onDeleteTag, isSelected, disabled }) => { +export const TagItem: React.FC = ({ item, onDeleteTag, isSelected, disabled }) => { const clickOnDeleteButton = () => onDeleteTag(item.id); return ( @@ -60,5 +60,3 @@ const TagItem: React.FC = ({ item, onDeleteTag, isSelected, disabled }) ); }; - -export default TagItem; diff --git a/airbyte-webapp/src/components/ui/TextArea/index.tsx b/airbyte-webapp/src/components/ui/TextArea/index.tsx index ad0ee5f42fe77e..cc37a6a03bbee8 100644 --- a/airbyte-webapp/src/components/ui/TextArea/index.tsx +++ b/airbyte-webapp/src/components/ui/TextArea/index.tsx @@ -1,5 +1 @@ -import { TextArea } from "./TextArea"; - export * from "./TextArea"; - -export default TextArea; diff --git a/airbyte-webapp/src/components/ui/Toast/ErrorSign.tsx b/airbyte-webapp/src/components/ui/Toast/ErrorSign.tsx index 6e9a4c50e7bc7a..d0e0e7fb7842ef 100644 --- a/airbyte-webapp/src/components/ui/Toast/ErrorSign.tsx +++ b/airbyte-webapp/src/components/ui/Toast/ErrorSign.tsx @@ -21,10 +21,8 @@ const ExclamationLight = styled(FontAwesomeIcon)` color: ${({ theme }) => theme.whiteColor}; `; -const ErrorSign: React.FC = () => ( +export const ErrorSign: React.FC = () => ( ); - -export default ErrorSign; diff --git a/airbyte-webapp/src/components/ui/Toast/SingletonCard.module.scss b/airbyte-webapp/src/components/ui/Toast/Toast.module.scss similarity index 100% rename from airbyte-webapp/src/components/ui/Toast/SingletonCard.module.scss rename to airbyte-webapp/src/components/ui/Toast/Toast.module.scss diff --git a/airbyte-webapp/src/components/ui/Toast/Toast.tsx b/airbyte-webapp/src/components/ui/Toast/Toast.tsx index 223ccd6ff66dd0..c0eec9417009a8 100644 --- a/airbyte-webapp/src/components/ui/Toast/Toast.tsx +++ b/airbyte-webapp/src/components/ui/Toast/Toast.tsx @@ -6,8 +6,8 @@ import styled, { keyframes } from "styled-components"; import { H5 } from "components/base/Titles"; import { Button } from "../Button"; -import ErrorSign from "./ErrorSign"; -import styles from "./SingletonCard.module.scss"; +import { ErrorSign } from "./ErrorSign"; +import styles from "./Toast.module.scss"; interface ToastProps { title: string | React.ReactNode; @@ -84,5 +84,3 @@ export const Toast: React.FC = (props) => ( )} ); - -export default Toast; diff --git a/airbyte-webapp/src/components/ui/index.tsx b/airbyte-webapp/src/components/ui/index.tsx deleted file mode 100644 index 7c4d3437afe024..00000000000000 --- a/airbyte-webapp/src/components/ui/index.tsx +++ /dev/null @@ -1,11 +0,0 @@ -export * from "./Button"; -export * from "./Card"; -export * from "./CheckBox"; -export * from "./DropDown"; -export * from "./Popout"; -export * from "./Input"; -export * from "./Multiselect"; -export * from "./RadioButton"; -export * from "./TagInput"; -export * from "./Switch"; -export * from "./TextArea"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx index d642dd683b2e78..f383b991cecbd4 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx @@ -5,7 +5,8 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; import * as yup from "yup"; -import { LabeledInput, Link, Button } from "components"; +import { LabeledInput, Link } from "components"; +import { Button } from "components/ui/Button"; import { useNotificationService } from "hooks/services/Notification/NotificationService"; import { useQuery } from "hooks/useQuery"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index 3ec8565d32c064..7e54e8344b157e 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -4,8 +4,9 @@ import { FormattedMessage, useIntl } from "react-intl"; import { NavigateOptions, To, useNavigate } from "react-router-dom"; import * as yup from "yup"; -import { LabeledInput, Link, Button } from "components"; +import { LabeledInput, Link } from "components"; import HeadTitle from "components/HeadTitle"; +import { Button } from "components/ui/Button"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useQuery } from "hooks/useQuery"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index d2c1c6011170ef..936bb9bf13e937 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -3,8 +3,9 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { LabeledInput, Link, Button } from "components"; +import { LabeledInput, Link } from "components"; import HeadTitle from "components/HeadTitle"; +import { Button } from "components/ui/Button"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useNotificationService } from "hooks/services/Notification/NotificationService"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx index 3cfb47e738aa4a..88de414482e005 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx @@ -5,7 +5,8 @@ import { useSearchParams } from "react-router-dom"; import styled from "styled-components"; import * as yup from "yup"; -import { LabeledInput, Link, Button } from "components"; +import { LabeledInput, Link } from "components"; +import { Button } from "components/ui/Button"; import { useConfig } from "config"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx index d738e815139861..b84a758fc32b94 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx @@ -2,9 +2,10 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; + +import { CloudRoutes } from "packages/cloud/cloudRoutes"; -import { CloudRoutes } from "../../../../cloudRoutes"; import styles from "./Header.module.scss"; interface HeaderProps { diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx index 7bee5c9ef4464c..d38c42c05b295f 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx @@ -6,7 +6,7 @@ import { useSearchParams } from "react-router-dom"; import { useEffectOnce } from "react-use"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { useConfig } from "config"; import { Action, Namespace } from "core/analytics"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index f6aaa699cc490e..c3222400e0bfd3 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { useMutation } from "react-query"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx index 50008d90cdcafe..9fd176599d70d0 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx @@ -3,8 +3,8 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; import { LabeledInput } from "components/LabeledInput"; +import { Button } from "components/ui/Button"; import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; import { FieldItem } from "packages/cloud/views/auth/components/FormComponents"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx index 3d52e8768b0e59..9234948301d8c3 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx @@ -3,8 +3,8 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { Button } from "components"; import { LabeledInput } from "components/LabeledInput"; +import { Button } from "components/ui/Button"; import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; import { RowFieldItem } from "packages/cloud/views/auth/components/FormComponents"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx index f3178b57b64808..3c1737e8a4dc3a 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx @@ -2,8 +2,8 @@ import { Field, FieldProps, Form, Formik } from "formik"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { Button } from "components"; import { LabeledInput } from "components/LabeledInput"; +import { Button } from "components/ui/Button"; import { FieldItem } from "packages/cloud/views/auth/components/FormComponents"; import FeedbackBlock from "pages/SettingsPage/components/FeedbackBlock"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index c62ecb15b83a3b..a958a032b432e6 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -6,9 +6,11 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button, DropDown, Input } from "components"; import { H5 } from "components/base/Titles"; import { Cell, Header, Row } from "components/SimpleTableComponents"; +import { Button } from "components/ui/Button"; +import { DropDown } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; import { Modal } from "components/ui/Modal"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx index d8f4379a2508ae..334a06eebabb90 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx @@ -3,8 +3,8 @@ import { FormattedMessage, useIntl } from "react-intl"; import { components, MenuListProps } from "react-select"; import styled from "styled-components"; -import { Popout } from "components"; -import { IDataItem } from "components/ui/DropDown/components/Option"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; +import { Popout } from "components/ui/Popout"; import { useListCloudWorkspacesAsync } from "packages/cloud/services/workspaces/CloudWorkspacesService"; import { useCurrentWorkspace, useWorkspaceService } from "services/workspaces/WorkspacesService"; @@ -66,7 +66,7 @@ const List = styled.div` } `; -type MenuWithRequestButtonProps = MenuListProps & { +type MenuWithRequestButtonProps = MenuListProps & { selectedWorkspace: string; }; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx index 1d674717c395a9..baf848c01adf92 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx @@ -4,7 +4,8 @@ import { FormattedMessage } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Input, Button } from "components"; +import { Button } from "components/ui/Button"; +import { Input } from "components/ui/Input"; interface CreateWorkspaceFormProps { onSubmit: (values: { name: string }) => Promise; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx index 664c178ffb8c3d..f658e35117282a 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx @@ -3,7 +3,8 @@ import { FormattedMessage } from "react-intl"; import { useToggle } from "react-use"; import styled from "styled-components"; -import { Button, Card } from "components"; +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; import CreateWorkspaceForm from "./CreateWorkspaceForm"; import styles from "./WorkspaceControl.module.scss"; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx index d444bc23f3362e..0fd292e4773579 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx @@ -4,9 +4,10 @@ import React, { Suspense } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button, LoadingPage, MainPageWithScroll } from "components"; +import { LoadingPage, MainPageWithScroll } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; +import { Button } from "components/ui/Button"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx index 8bb38d8df64fb8..c9b49a51531078 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx @@ -2,7 +2,7 @@ import { faPenToSquare } from "@fortawesome/free-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { ChangeEvent, useState } from "react"; -import { Input } from "components"; +import { Input } from "components/ui/Input"; import { Text } from "components/ui/Text"; import { buildConnectionUpdate } from "core/domain/connection"; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx index 595bcd2c05ec92..8da2df5e7daf42 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { useUpdateEffect } from "react-use"; import styled from "styled-components"; -import { Switch } from "components"; +import { Switch } from "components/ui/Switch"; import { Action, Namespace } from "core/analytics"; import { buildConnectionUpdate } from "core/domain/connection"; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx index 2552b134030507..b01fe9c1009842 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx @@ -4,9 +4,11 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button, ControlLabels, DropDown } from "components"; import { ConnectorIcon } from "components/ConnectorIcon"; +import { ControlLabels } from "components/LabeledControl"; +import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; +import { DropDown } from "components/ui/DropDown"; import { useDestinationList } from "hooks/services/useDestinationHook"; import { useSourceList } from "hooks/services/useSourceHook"; diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx index 0aa6aaac0b5331..f6029b47780ccd 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx @@ -4,9 +4,10 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button, MainPageWithScroll } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; +import { MainPageWithScroll } from "components/MainPageWithScroll"; +import { Button } from "components/ui/Button"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx index 736ed8f6f9a171..a87d607d2f10e0 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx @@ -2,13 +2,14 @@ import React, { Suspense, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Route, Routes, useNavigate, useParams } from "react-router-dom"; -import { DropDownRow, LoadingPage } from "components"; +import { LoadingPage } from "components"; import ApiErrorBoundary from "components/ApiErrorBoundary"; import { ItemTabs, StepsTypes, TableItemTitle } from "components/ConnectorBlocks"; import { ConnectorIcon } from "components/ConnectorIcon"; import HeadTitle from "components/HeadTitle"; import Placeholder, { ResourceTypes } from "components/Placeholder"; import { Breadcrumbs } from "components/ui/Breadcrumbs"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -72,7 +73,7 @@ const DestinationItemPage: React.FC = () => { [sources, sourceDefinitions] ); - const onSelect = (data: DropDownRow.IDataItem) => { + const onSelect = (data: DropDownOptionDataItem) => { const path = `../${RoutePaths.ConnectionNew}`; const state = data.value === "create-new-item" diff --git a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx index 1e0e83e5be7b65..33010f626b0fcd 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx @@ -4,10 +4,10 @@ import { useNavigate } from "react-router-dom"; import { useEffectOnce } from "react-use"; import styled from "styled-components"; -import { Button } from "components"; import ApiErrorBoundary from "components/ApiErrorBoundary"; import HeadTitle from "components/HeadTitle"; import LoadingPage from "components/LoadingPage"; +import { Button } from "components/ui/Button"; import { useAnalyticsService, useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useWorkspace from "hooks/services/useWorkspace"; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx index 492f5c9b2fc2ca..77cef9f8c66645 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx @@ -5,13 +5,13 @@ import { FormattedMessage } from "react-intl"; import styled, { keyframes } from "styled-components"; import Link from "components/Link"; -import { Button } from "components/ui"; +import { Button } from "components/ui/Button"; import { Text } from "components/ui/Text"; +import { JobStatus, WebBackendConnectionRead } from "core/request/AirbyteClient"; import Status from "core/statuses"; +import { RoutePaths } from "pages/routePaths"; -import { JobStatus, WebBackendConnectionRead } from "../../../core/request/AirbyteClient"; -import { RoutePaths } from "../../routePaths"; import styles from "./ProgressBlock.module.scss"; const run = keyframes` diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/WelcomeStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/WelcomeStep.tsx index 3fd09ff0d720db..b2b151b6d6decc 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/WelcomeStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/WelcomeStep.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { useConfig } from "config"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/LogsContent.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/LogsContent.tsx index 380a616879cd3d..3de89859cfb6af 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/LogsContent.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/LogsContent.tsx @@ -3,7 +3,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { LogType } from "core/domain/logs/types"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx index 419e870dd6652f..8b6908d9f2c5e5 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx @@ -4,7 +4,7 @@ import React, { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { RoutePaths } from "pages/routePaths"; import { useCreateDestinationDefinition } from "services/connector/DestinationDefinitionService"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx index 680f05e22dfddc..aaefc1f98ef48b 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx @@ -4,7 +4,8 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button, LabeledInput, Link, StatusIcon } from "components"; +import { LabeledInput, Link, StatusIcon } from "components"; +import { Button } from "components/ui/Button"; import { Modal } from "components/ui/Modal"; import { useConfig } from "config"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx index 90347d80cf34b1..3f76768c2778e5 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx @@ -4,7 +4,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import styles from "./UpgradeAllButton.module.scss"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx index c16b5bff81828c..8be0cc07d6afbb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx @@ -3,7 +3,8 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; -import { Input, Button } from "components"; +import { Button } from "components/ui/Button"; +import { Input } from "components/ui/Input"; import { DEV_IMAGE_TAG } from "core/domain/connector/constants"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx index bfb6bdeb2f1be4..c09f9f97a717f6 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx @@ -4,8 +4,10 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Label, Input, LabeledSwitch, Button } from "components"; +import { Label, LabeledSwitch } from "components"; import { Row, Cell } from "components/SimpleTableComponents"; +import { Button } from "components/ui/Button"; +import { Input } from "components/ui/Input"; import { WebhookPayload } from "hooks/services/useWorkspace"; import { equal } from "utils/objects"; diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx index 41abb9f3acd8d3..5adacbac7bb84c 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx @@ -4,9 +4,10 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button, MainPageWithScroll } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; +import { MainPageWithScroll } from "components/MainPageWithScroll"; +import { Button } from "components/ui/Button"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx index d046d22c019a9c..03949f1f898257 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx @@ -2,7 +2,6 @@ import React, { Suspense, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Route, Routes, useNavigate, useParams } from "react-router-dom"; -import { DropDownRow } from "components"; import ApiErrorBoundary from "components/ApiErrorBoundary"; import { ItemTabs, StepsTypes, TableItemTitle } from "components/ConnectorBlocks"; import { ConnectorIcon } from "components/ConnectorIcon"; @@ -10,6 +9,7 @@ import HeadTitle from "components/HeadTitle"; import LoadingPage from "components/LoadingPage"; import Placeholder, { ResourceTypes } from "components/Placeholder"; import { Breadcrumbs } from "components/ui/Breadcrumbs"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -73,7 +73,7 @@ const SourceItemPage: React.FC = () => { navigate(path); }; - const onSelect = (data: DropDownRow.IDataItem) => { + const onSelect = (data: DropDownOptionDataItem) => { const path = `../${RoutePaths.ConnectionNew}`; const state = data.value === "create-new-item" diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.tsx index efb46608409c95..0d01ad9770bac7 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.tsx @@ -1,7 +1,7 @@ import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { AirbyteCatalog, CatalogDiff } from "core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx index 58ef3ce9144f89..ab755a16a4441a 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx @@ -2,7 +2,7 @@ import { FormikErrors, getIn } from "formik"; import React, { memo, useCallback, useMemo } from "react"; import { useToggle } from "react-use"; -import { DropDownRow } from "components"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; import { SyncSchemaField, SyncSchemaFieldObject, SyncSchemaStream } from "core/domain/catalog"; import { traverseSchemaToField } from "core/domain/catalog/fieldUtil"; @@ -54,7 +54,7 @@ const CatalogSectionInner: React.FC = ({ ); const onSelectSyncMode = useCallback( - (data: DropDownRow.IDataItem) => updateStreamWithConfig(data.value), + (data: DropDownOptionDataItem) => updateStreamWithConfig(data.value), [updateStreamWithConfig] ); diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx index 026029f0d34b3c..125833bfe2a7d9 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx @@ -1,13 +1,15 @@ import React, { memo } from "react"; import styled from "styled-components"; -import { Cell, CheckBox, RadioButton } from "components"; +import { Cell } from "components/SimpleTableComponents"; +import { CheckBox } from "components/ui/CheckBox"; +import { RadioButton } from "components/ui/RadioButton"; import { SyncSchemaField } from "core/domain/catalog"; import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; import { equal } from "utils/objects"; +import { useTranslateDataType } from "utils/useTranslateDataType"; -import { useTranslateDataType } from "../../../utils/useTranslateDataType"; import DataTypeCell from "./components/DataTypeCell"; import { pathDisplayName } from "./components/PathPopout"; import { NameContainer } from "./styles"; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx index 89b9942df304e8..04bb2900619b26 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx @@ -5,7 +5,10 @@ import React, { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Cell, CheckBox, DropDownRow, Row, Switch } from "components"; +import { Cell, Row } from "components"; +import { CheckBox } from "components/ui/CheckBox"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; +import { Switch } from "components/ui/Switch"; import { Path, SyncSchemaField, SyncSchemaStream } from "core/domain/catalog"; import { DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; @@ -34,7 +37,7 @@ interface StreamHeaderProps { availableSyncModes: Array<{ value: SyncSchema; }>; - onSelectSyncMode: (selectedMode: DropDownRow.IDataItem) => void; + onSelectSyncMode: (selectedMode: DropDownOptionDataItem) => void; onSelectStream: () => void; primitiveFields: SyncSchemaField[]; pkType: IndexerType; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx index 58aabe5a97b866..db1b31d85484bc 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx @@ -3,7 +3,9 @@ import React, { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button, Cell, Header, Switch } from "components"; +import { Cell, Header } from "components"; +import { Button } from "components/ui/Button"; +import { Switch } from "components/ui/Switch"; import { SyncSchemaField, SyncSchemaFieldObject, SyncSchemaStream, traverseSchemaToField } from "core/domain/catalog"; import { DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx index 50297f1bd3a0e3..9dee9dd8717890 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx @@ -3,10 +3,14 @@ import { FormattedMessage } from "react-intl"; import { components, ControlProps } from "react-select"; import styled from "styled-components"; -import { DropDown, DropdownProps } from "components"; -import { IDataItem, OptionView } from "components/ui/DropDown/components/Option"; -import { IProps } from "components/ui/DropDown/components/SingleValue"; -import Text from "components/ui/DropDown/components/Text"; +import { + DropDown, + DropdownProps, + DropDownText, + DropDownOptionDataItem, + OptionView, + SingleValueProps, +} from "components/ui/DropDown"; const ValueView = styled(components.SingleValue)` display: flex; @@ -24,10 +28,10 @@ const Separator = styled.div` padding: 0 5px; `; -const SingleValue: React.FC> = (props) => { +const SingleValue: React.FC> = (props) => { const { syncMode, destinationSyncMode } = props.data?.value; return ( - +
@@ -37,7 +41,7 @@ const SingleValue: React.FC> = (props) => {
-
+ ); }; @@ -52,7 +56,7 @@ const OptionContent = styled(OptionView)` font-size: 12px; `; -const DropdownControl = styled(components.Control)>` +const DropdownControl = styled(components.Control)>` &.react-select__control { // background: ${({ theme }) => theme.greyColor20}; // TODO: fix theme diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx index a82c391fc73860..b2c221e4d5fd99 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx @@ -2,7 +2,8 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button, StatusIcon } from "components"; +import { StatusIcon } from "components/StatusIcon"; +import { Button } from "components/ui/Button"; import { Spinner } from "components/ui/Spinner"; interface CreateControlsProps { diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/EditControls.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/EditControls.tsx index c3835093644ed6..fc78785d645827 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/EditControls.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/EditControls.tsx @@ -1,7 +1,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import styles from "./EditControls.module.scss"; import { ResponseMessage } from "./ResponseMessage"; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/NamespaceDefinitionField.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/NamespaceDefinitionField.tsx index b1ff8ab1c5bb8f..0d304971a8490c 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/NamespaceDefinitionField.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/NamespaceDefinitionField.tsx @@ -2,7 +2,8 @@ import { FieldProps, useField } from "formik"; import React from "react"; import { FormattedMessage } from "react-intl"; -import { ControlLabels, DropDown } from "components"; +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; import { NamespaceDefinitionType } from "../../../../core/request/AirbyteClient"; import styles from "./NamespaceDefinitionField.module.scss"; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx index 18d883b47c9db0..3bdf5c87b15367 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/ScheduleField.tsx @@ -2,8 +2,9 @@ import { Field, FieldInputProps, FieldProps, FormikProps } from "formik"; import { ChangeEvent, useCallback, useMemo } from "react"; import { useIntl } from "react-intl"; -import { ControlLabels, DropDown, DropDownRow, Input, Link } from "components"; -import { IDataItem } from "components/ui/DropDown/components/Option"; +import { ControlLabels, Link } from "components"; +import { DropDown, DropDownOptionDataItem } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; import { Action, Namespace } from "core/analytics"; import { ConnectionScheduleData, ConnectionScheduleType } from "core/request/AirbyteClient"; @@ -29,7 +30,7 @@ const ScheduleField: React.FC = () => { const analyticsService = useAnalyticsService(); const onDropDownSelect = useCallback( - (item: IDataItem | null) => { + (item: DropDownOptionDataItem | null) => { const enabledStreams = connection.syncCatalog.streams.filter((stream) => stream.config?.selected).length; if (item) { @@ -57,7 +58,7 @@ const ScheduleField: React.FC = () => { ] ); - const onScheduleChange = (item: DropDownRow.IDataItem, form: FormikProps) => { + const onScheduleChange = (item: DropDownOptionDataItem, form: FormikProps) => { onDropDownSelect?.(item); let scheduleData: ConnectionScheduleData; @@ -104,7 +105,7 @@ const ScheduleField: React.FC = () => { const getZoneValue = (currentSelectedZone = "UTC") => currentSelectedZone; const onCronChange = ( - event: DropDownRow.IDataItem | ChangeEvent, + event: DropDownOptionDataItem | ChangeEvent, field: FieldInputProps, form: FormikProps, key: string @@ -112,8 +113,8 @@ const ScheduleField: React.FC = () => { form.setFieldValue(field.name, { cron: { ...field.value?.cron, - [key]: (event as DropDownRow.IDataItem).value - ? (event as DropDownRow.IDataItem).value + [key]: (event as DropDownOptionDataItem).value + ? (event as DropDownOptionDataItem).value : (event as ChangeEvent).currentTarget.value, }, }); @@ -200,7 +201,7 @@ const ScheduleField: React.FC = () => { className={styles.cronZonesDropdown} options={cronTimeZones} value={getZoneValue(field.value?.cron?.cronTimeZone)} - onChange={(item: DropDownRow.IDataItem) => onCronChange(item, field, form, "cronTimeZone")} + onChange={(item: DropDownOptionDataItem) => onCronChange(item, field, form, "cronTimeZone")} /> diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx index e92f3c28598e27..56ad662fcbc9cc 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx @@ -2,7 +2,7 @@ import React from "react"; import { useIntl } from "react-intl"; import styled from "styled-components"; -import { Input } from "components"; +import { Input } from "components/ui/Input"; interface SearchProps { onSearch: (value: string) => void; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx index 57d28f56ed95b8..40329d44f7f7fa 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx @@ -1,4 +1,4 @@ -import { Card } from "components/ui"; +import { Card } from "components/ui/Card"; import { Text } from "components/ui/Text"; import styles from "./Section.module.scss"; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index db5e73a7dc628f..a7b25acaf04b1d 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -2,7 +2,7 @@ import { useMemo } from "react"; import { useIntl } from "react-intl"; import * as yup from "yup"; -import { DropDownRow } from "components"; +import { DropDownOptionDataItem } from "components/ui/DropDown"; import { frequencyConfig } from "config/frequencyConfig"; import { SyncSchema } from "core/domain/catalog"; @@ -294,7 +294,7 @@ export const useInitialValues = ( export const useFrequencyDropdownData = ( additionalFrequency: WebBackendConnectionRead["scheduleData"] -): DropDownRow.IDataItem[] => { +): DropDownOptionDataItem[] => { const { formatMessage } = useIntl(); return useMemo(() => { diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index afad0341865ad5..bf39b6a4102892 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -1,8 +1,8 @@ import React, { useEffect, useState } from "react"; import { FormattedMessage } from "react-intl"; -import { Card } from "components"; import { JobItem } from "components/JobItem/JobItem"; +import { Card } from "components/ui/Card"; import { Action, Namespace } from "core/analytics"; import { Connector, ConnectorT } from "core/domain/connector"; diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx index 9b62556ac96421..6f641a334b7712 100644 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx @@ -4,7 +4,10 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button, ControlLabels, DropDown, Input } from "components"; +import { ControlLabels } from "components/LabeledControl"; +import { Button } from "components/ui/Button"; +import { DropDown } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; import { Values } from "../types"; import styles from "./ConnectorForm.module.scss"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorNameControl.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorNameControl.tsx index 2143c43b2d2f13..1773d7fb0ce8c1 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorNameControl.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorNameControl.tsx @@ -2,7 +2,7 @@ import { useField } from "formik"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { Input } from "components"; +import { Input } from "components/ui/Input"; import { FormBaseItem } from "core/form/types"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx index a6a517498ad11b..3fe8921b2db1c8 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx @@ -5,15 +5,18 @@ import { components } from "react-select"; import { MenuListProps } from "react-select"; import styled from "styled-components"; -import { ControlLabels, DropDown, DropDownRow } from "components"; import { ConnectorIcon } from "components/ConnectorIcon"; import { GAIcon } from "components/icons/GAIcon"; -import { IDataItem, IProps as OptionProps, OptionView } from "components/ui/DropDown/components/Option"; +import { ControlLabels } from "components/LabeledControl"; import { - Icon as SingleValueIcon, - IProps as SingleValueProps, - ItemView as SingleValueView, -} from "components/ui/DropDown/components/SingleValue"; + DropDown, + DropDownOptionDataItem, + DropDownOptionProps, + OptionView, + SingleValueIcon, + SingleValueProps, + SingleValueView, +} from "components/ui/DropDown"; import { Action, Namespace } from "core/analytics"; import { Connector, ConnectorDefinition } from "core/domain/connector"; @@ -78,7 +81,7 @@ const SingleValueContent = styled(components.SingleValue)` `; // eslint-disable-next-line @typescript-eslint/no-explicit-any -type MenuWithRequestButtonProps = MenuListProps & { selectProps: any }; +type MenuWithRequestButtonProps = MenuListProps & { selectProps: any }; /** * Returns the order for a specific release stage label. This will define @@ -124,7 +127,7 @@ const StageLabel: React.FC<{ releaseStage?: ReleaseStage }> = ({ releaseStage }) ); }; -const Option: React.FC = (props) => { +const Option: React.FC = (props) => { return ( @@ -224,7 +227,7 @@ const ConnectorServiceTypeControl: React.FC = ); const handleSelect = useCallback( - (item: DropDownRow.IDataItem | null) => { + (item: DropDownOptionDataItem | null) => { if (item) { setValue(item.value); if (onChangeServiceType) { diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/CreateControls.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/CreateControls.tsx index 2b389c7495ba94..d0f60d62834bd9 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/CreateControls.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/CreateControls.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import styles from "./CreateControls.module.scss"; import { TestingConnectionError, FetchingConnectorError } from "./TestingConnectionError"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/EditControls.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/EditControls.tsx index 49e83379ec7e86..642a16f32a4e9d 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/EditControls.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/EditControls.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { useServiceForm } from "../serviceFormContext"; import styles from "./EditControls.module.scss"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/FrequentlyUsedDestinations/__snapshots__/FrequentlyUsedDestinations.test.tsx.snap b/airbyte-webapp/src/views/Connector/ServiceForm/components/FrequentlyUsedDestinations/__snapshots__/FrequentlyUsedDestinations.test.tsx.snap index 3270eeb4fa50de..a83d88e7e21757 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/FrequentlyUsedDestinations/__snapshots__/FrequentlyUsedDestinations.test.tsx.snap +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/FrequentlyUsedDestinations/__snapshots__/FrequentlyUsedDestinations.test.tsx.snap @@ -74,7 +74,7 @@ exports[` should renders with mock data without cr > @@ -93,7 +93,7 @@ exports[` should renders with mock data without cr class="container" >
Alpha
@@ -125,7 +125,7 @@ exports[` should renders with mock data without cr > @@ -188,7 +188,7 @@ exports[` should renders with mock data without cr > @@ -230,7 +230,7 @@ exports[` should renders with mock data without cr > @@ -272,7 +272,7 @@ exports[` should renders with mock data without cr > @@ -291,7 +291,7 @@ exports[` should renders with mock data without cr class="container" >
Alpha
diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/ConfirmationControl.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/ConfirmationControl.tsx index a4ced701f32fdf..ff4adb9f71ae48 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/ConfirmationControl.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/ConfirmationControl.tsx @@ -2,7 +2,7 @@ import React, { useRef } from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; const ComponentContainer = styled.div` display: flex; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx index 9786a7e130736e..4cf505c8c7f7a3 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx @@ -1,7 +1,11 @@ import { FieldArray, useField } from "formik"; import React from "react"; -import { DropDown, Input, Multiselect, TextArea, TagInput } from "components"; +import { DropDown } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; +import { Multiselect } from "components/ui/Multiselect"; +import { TagInput } from "components/ui/TagInput"; +import { TextArea } from "components/ui/TextArea"; import { FormBaseItem } from "core/form/types"; import { isDefined } from "utils/common"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/PropertyLabel.stories.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/PropertyLabel.stories.tsx index e3158787e4aa91..ff77000ea784be 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/PropertyLabel.stories.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/PropertyLabel.stories.tsx @@ -1,6 +1,7 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; -import { Card, Input } from "components"; +import { Card } from "components/ui/Card"; +import { Input } from "components/ui/Input"; import { PropertyLabel } from "./PropertyLabel"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx index 595fc5f543279c..18c0c09e666f10 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx @@ -2,8 +2,7 @@ import { useFormikContext, setIn } from "formik"; import React, { useCallback, useMemo } from "react"; import GroupControls from "components/GroupControls"; -import { DropDown } from "components/ui/DropDown"; -import { IDataItem } from "components/ui/DropDown/components/Option"; +import { DropDown, DropDownOptionDataItem } from "components/ui/DropDown"; import { FormBlock, FormConditionItem } from "core/form/types"; import { isDefined } from "utils/common"; @@ -30,7 +29,7 @@ export const ConditionSection: React.FC = ({ formField, p const currentlySelectedCondition = widgetsInfo[formField.path]?.selectedItem; const onOptionChange = useCallback( - (selectedItem: IDataItem) => { + (selectedItem: DropDownOptionDataItem) => { const newSelectedPath = formField.conditions[selectedItem.value]; const newValues = diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx index 5a884f21cb143e..9ff34bfd9c6125 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx @@ -2,7 +2,7 @@ import classnames from "classnames"; import React from "react"; import { FormattedMessage } from "react-intl"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { Text } from "components/ui/Text"; import { ConnectorSpecification } from "core/domain/connector"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/StartWithDestination.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/StartWithDestination.tsx index 7828e8197642f1..bd1af75aa2760f 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/StartWithDestination.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/StartWithDestination.tsx @@ -2,7 +2,8 @@ import { useField } from "formik"; import React from "react"; import { useIntl } from "react-intl"; -import { Card, ConnectorCard } from "components"; +import { ConnectorCard } from "components/ConnectorCard"; +import { Card } from "components/ui/Card"; import { DestinationConnectorCard } from "../../types"; import styles from "./StartWithDestination.module.scss"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/__snapshots__/StartWithDestination.test.tsx.snap b/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/__snapshots__/StartWithDestination.test.tsx.snap index e4704e8a81c54e..6555679b3e5ab4 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/__snapshots__/StartWithDestination.test.tsx.snap +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/StartWithDestination/__snapshots__/StartWithDestination.test.tsx.snap @@ -22,7 +22,7 @@ exports[` should renders without crash with provided pr > @@ -41,7 +41,7 @@ exports[` should renders without crash with provided pr class="container" >
Alpha
diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/TestingConnectionSpinner.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/TestingConnectionSpinner.tsx index 0741e7ff912b57..634d30f1952b13 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/TestingConnectionSpinner.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/TestingConnectionSpinner.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { ProgressBar } from "components/ui/ProgressBar"; import styles from "./TestingConnectionSpinner.module.scss"; diff --git a/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx b/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx index bac20190b0aaf9..2e7abc2bb32048 100644 --- a/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx +++ b/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx @@ -4,10 +4,10 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button } from "components"; import Label from "components/Label"; import LabeledInput from "components/LabeledInput"; import { LabeledSwitch } from "components/LabeledSwitch"; +import { Button } from "components/ui/Button"; import { useConfig } from "config"; diff --git a/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx b/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx index ca068c624dcbe5..96315c2627a885 100644 --- a/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx +++ b/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Button } from "components"; +import { Button } from "components/ui/Button"; import { Spinner } from "components/ui/Spinner"; interface IProps { From dfaf7d860be28d6b519920b948db26598b964464 Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Wed, 5 Oct 2022 12:14:40 +0200 Subject: [PATCH 30/32] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Replace=20m?= =?UTF-8?q?ultiline=20+=20hidden=20connector=20field=20with=20`SecretTextA?= =?UTF-8?q?rea`=20component=20(#16539)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add SecretTextArea component * Add TextInputContainer to manage the look and feel of inputs and text area * Update multiline + secret controls to use it * Fix SecretTextArea story * Add .fs-exclude class to SecretTextArea and Password input * Use scss color * Add tests for SecretTextArea component * Update SecretTextArea testIds * Remove specificity in secrettextarea rule * Update type to be compatible with React 18 * Move SecretTextArea and TextInputContainer to components/ui --- .../src/components/ui/Input/Input.tsx | 1 + .../SecretTextArea/SecretTextArea.module.scss | 21 +++++ .../ui/SecretTextArea/SecretTextArea.test.tsx | 85 +++++++++++++++++ .../ui/SecretTextArea/SecretTextArea.tsx | 93 +++++++++++++++++++ .../ui/SecretTextArea/index.stories.tsx | 27 ++++++ .../src/components/ui/SecretTextArea/index.ts | 1 + .../TextInputContainer.module.scss | 73 +++++++++++++++ .../TextInputContainer/TextInputContainer.tsx | 45 +++++++++ .../ui/TextInputContainer/index.stories.tsx | 20 ++++ .../components/ui/TextInputContainer/index.ts | 1 + airbyte-webapp/src/locales/en.json | 1 + .../components/Property/Control.tsx | 10 +- 12 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.module.scss create mode 100644 airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.test.tsx create mode 100644 airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.tsx create mode 100644 airbyte-webapp/src/components/ui/SecretTextArea/index.stories.tsx create mode 100644 airbyte-webapp/src/components/ui/SecretTextArea/index.ts create mode 100644 airbyte-webapp/src/components/ui/TextInputContainer/TextInputContainer.module.scss create mode 100644 airbyte-webapp/src/components/ui/TextInputContainer/TextInputContainer.tsx create mode 100644 airbyte-webapp/src/components/ui/TextInputContainer/index.stories.tsx create mode 100644 airbyte-webapp/src/components/ui/TextInputContainer/index.ts diff --git a/airbyte-webapp/src/components/ui/Input/Input.tsx b/airbyte-webapp/src/components/ui/Input/Input.tsx index 98f458e48bb75f..40acaf84d0a8d1 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.tsx +++ b/airbyte-webapp/src/components/ui/Input/Input.tsx @@ -89,6 +89,7 @@ export const Input: React.FC = ({ light, error, ...props }) => { { [styles.disabled]: props.disabled, [styles.password]: isPassword, + "fs-exclude": isPassword, }, props.className )} diff --git a/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.module.scss b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.module.scss new file mode 100644 index 00000000000000..53a715d88d952d --- /dev/null +++ b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.module.scss @@ -0,0 +1,21 @@ +@use "../../../scss/colors"; + +.toggleVisibilityButton { + width: 100%; + line-height: 1; + color: colors.$grey-300; + font-style: italic; + margin: 0; + padding: 0; + border: none; + background: none; + cursor: text; +} + +.textarea { + overflow: auto; +} + +.passwordInput { + display: none; +} diff --git a/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.test.tsx b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.test.tsx new file mode 100644 index 00000000000000..d97da1346a8f57 --- /dev/null +++ b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.test.tsx @@ -0,0 +1,85 @@ +import userEvent from "@testing-library/user-event"; +import { act } from "react-dom/test-utils"; +import { render } from "test-utils/testutils"; + +import { SecretTextArea } from "./SecretTextArea"; + +// eslint-disable-next-line @typescript-eslint/no-empty-function +const emptyFn = () => {}; + +describe("SecretTextArea", () => { + it("renders textarea when there is no initial value", async () => { + const { queryByTestId, container } = await render(); + + expect(container.querySelector("textarea")).toBeInTheDocument(); + expect(queryByTestId("secretTextArea-visibilityButton")).not.toBeInTheDocument(); + expect(container.querySelector('input[type="password"]')).not.toBeInTheDocument(); + }); + + it("renders on hidden input when there is an initial value", async () => { + const value = "Here is my secret text"; + const { getByTestId, queryByTestId, container } = await render(); + + expect(container.querySelector("textarea")).not.toBeInTheDocument(); + expect(queryByTestId("secretTextArea-visibilityButton")).toBeInTheDocument(); + + const input = getByTestId("secretTextArea-input"); + expect(input).toHaveAttribute("type", "password"); + expect(input).toHaveAttribute("aria-hidden"); + expect(input).toHaveValue(value); + }); + + it("renders disabled when disabled is set", async () => { + const { getByTestId } = await render(); + + expect(getByTestId("textInputContainer")).toHaveClass("disabled"); + expect(getByTestId("secretTextArea-textarea")).toBeDisabled(); + }); + + it("renders disabled when disabled is set and with initial value", async () => { + const value = "Here is my secret text"; + const { getByTestId } = await render(); + + expect(getByTestId("textInputContainer")).toHaveClass("disabled"); + expect(getByTestId("secretTextArea-visibilityButton")).toBeDisabled(); + }); + + it("calls onChange handler when typing", async () => { + const onChange = jest.fn(); + const value = "Here is my secret text"; + const { getByTestId } = await render(); + + const textarea = getByTestId("secretTextArea-textarea"); + + userEvent.type(textarea, value); + + expect(onChange).toBeCalledTimes(value.length); + }); + + it("renders on textarea when clicked visibility button", async () => { + const value = "Here is my secret text"; + const { getByTestId, container } = await render(); + + userEvent.click(getByTestId("secretTextArea-visibilityButton")); + + expect(getByTestId("secretTextArea-textarea")).toHaveFocus(); + expect(getByTestId("secretTextArea-textarea")).toHaveValue(value); + expect(container.querySelector('input[type="password"]')).not.toBeInTheDocument(); + }); + + it("renders on password input when clicking away from visibility area", async () => { + const value = "Here is my secret text"; + const { queryByTestId, getByTestId, container } = await render(); + + userEvent.click(getByTestId("secretTextArea-visibilityButton")); + expect(getByTestId("secretTextArea-textarea")).toHaveFocus(); + + act(() => { + getByTestId("secretTextArea-textarea").blur(); + }); + + expect(container.querySelector("textarea")).not.toBeInTheDocument(); + expect(queryByTestId("secretTextArea-visibilityButton")).toBeInTheDocument(); + expect(container.querySelector('input[type="password"]')).toHaveValue(value); + }); +}); diff --git a/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.tsx b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.tsx new file mode 100644 index 00000000000000..90aaa4fbbe2070 --- /dev/null +++ b/airbyte-webapp/src/components/ui/SecretTextArea/SecretTextArea.tsx @@ -0,0 +1,93 @@ +import { faEye } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import classNames from "classnames"; +import { useMemo, useRef } from "react"; +import { FormattedMessage } from "react-intl"; +import { useToggle, useUpdateEffect } from "react-use"; + +import { TextInputContainer, TextInputContainerProps } from "../TextInputContainer"; +import styles from "./SecretTextArea.module.scss"; + +interface SecretTextAreaProps + extends Omit, + React.TextareaHTMLAttributes {} + +export const SecretTextArea: React.FC = ({ + name, + disabled, + value, + onMouseUp, + onBlur, + error, + light, + ...textAreaProps +}) => { + const hasValue = useMemo(() => !!value && String(value).trim().length > 0, [value]); + const [isContentVisible, toggleIsContentVisible] = useToggle(!hasValue); + const textAreaRef = useRef(null); + const textAreaHeightRef = useRef((textAreaProps.rows ?? 1) * 20 + 14); + + useUpdateEffect(() => { + if (isContentVisible && textAreaRef.current) { + textAreaRef.current.focus(); + const selectionStart = value ? String(value).length : 0; + textAreaRef.current.setSelectionRange(selectionStart, selectionStart); + } + }, [isContentVisible]); + + return ( + + {isContentVisible ? ( +