From 546adcb7e04cdad9eefac35d1a35615aaa05c0b2 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Mon, 12 Jun 2023 18:01:13 -0300 Subject: [PATCH 01/18] :bug: Source Dixa: fix tests and conversation stream (#27250) * fix tests and conversation stream * fix eof conversation export stream * fix unit test * fix: semantic versioning correction for new stream --------- Co-authored-by: sajarin --- .../connectors/source-dixa/Dockerfile | 2 +- .../source-dixa/acceptance-test-config.yml | 2 +- .../integration_tests/sample_config.json | 2 +- .../connectors/source-dixa/metadata.yaml | 2 +- .../connectors/source-dixa/setup.py | 2 +- .../schemas/conversation_export.json | 38 ++++++++++++++++++- .../source-dixa/source_dixa/source.py | 12 +++--- .../source-dixa/source_dixa/spec.json | 2 +- .../source-dixa/unit_tests/unit_test.py | 4 +- docs/integrations/sources/dixa.md | 1 + 10 files changed, 52 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-dixa/Dockerfile b/airbyte-integrations/connectors/source-dixa/Dockerfile index 93cb4d948a8ef..e902f7d6f7d6c 100644 --- a/airbyte-integrations/connectors/source-dixa/Dockerfile +++ b/airbyte-integrations/connectors/source-dixa/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-dixa diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml index a3a615fb5a349..30ff974f5d741 100644 --- a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json index 580eb448f5d6d..efe861f2078b2 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { "api_token": "TOKEN", "start_date": "2020-01-01", - "batch_size": "31" + "batch_size": 31 } diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index 40b717ae6a3d4..1a76236a834e7 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-dixa githubIssueLabel: source-dixa icon: dixa.svg diff --git a/airbyte-integrations/connectors/source-dixa/setup.py b/airbyte-integrations/connectors/source-dixa/setup.py index 72e5ddbe8e3dc..40110e59e2295 100644 --- a/airbyte-integrations/connectors/source-dixa/setup.py +++ b/airbyte-integrations/connectors/source-dixa/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json index bd88556883f58..6ee94ca78602f 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json @@ -1,6 +1,8 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Conversation Export", + "additionalProperties": true, "type": ["null", "object"], - "required": ["id", "created_at", "initial_channel", "requester_id"], "properties": { "id": { "type": ["null", "integer"] @@ -145,6 +147,13 @@ "type": ["null", "string"] } }, + "tags_info": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, "conversation_wrapup_notes": { "type": ["null", "array"], "items": { @@ -163,6 +172,9 @@ "status": { "type": ["null", "string"] }, + "transferee_number": { + "type": ["null", "string"] + }, "updated_at": { "type": ["null", "integer"] }, @@ -180,6 +192,30 @@ }, "anonymized_at": { "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "attribute_id": { + "type": ["null", "string"] + }, + "identifier": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "array", "string"] + }, + "data_type": { + "type": ["null", "string"] + }, + "archived": { + "type": ["null", "boolean"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py index 63f2cb8a05a00..8f06896728813 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py @@ -109,13 +109,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> """ Check connectivity using one day's worth of data. """ - config["authenticator"] = TokenAuthenticator(token=config["api_token"]) - stream = ConversationExport(config) - # using 1 day batch size for slices. - stream.batch_size = 1 - # use the first slice from stream_slices list - stream_slice = stream.stream_slices()[0] try: + config["authenticator"] = TokenAuthenticator(token=config["api_token"]) + stream = ConversationExport(config) + # using 1 day batch size for slices. + stream.batch_size = 1 + # use the first slice from stream_slices list + stream_slice = stream.stream_slices()[0] list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) return True, None except Exception as e: diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json index 016b237406a55..d4bfe2b034174 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json @@ -5,7 +5,7 @@ "title": "Dixa Spec", "type": "object", "required": ["api_token", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "api_token": { "type": "string", diff --git a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py index 61ad3b2038cfe..cbd78e67e68c0 100644 --- a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py @@ -53,8 +53,8 @@ def test_stream_slices_without_state(conversation_export): conversation_export.end_timestamp = 1625259600000 # 2021-07-03 00:00:00 + 1 ms expected_slices = [ - {"updated_after": 1625097600000, "updated_before": 1625184000000}, - {"updated_after": 1625184000000, "updated_before": 1625259600000}, + {'updated_after': 1625097600000, 'updated_before': 1625184000000}, + {'updated_after': 1625184000000, 'updated_before': 1625259600000}, ] actual_slices = conversation_export.stream_slices() diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md index c1a1b1c961bf6..ced877c4a6b37 100644 --- a/docs/integrations/sources/dixa.md +++ b/docs/integrations/sources/dixa.md @@ -51,6 +51,7 @@ When using the connector, keep in mind that increasing the `batch_size` paramete | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------- | +| 0.2.0 | 2023-06-08 | [25103](https://github.com/airbytehq/airbyte/pull/25103) | Add fields to `conversation_export` stream | | 0.1.3 | 2022-07-07 | [14437](https://github.com/airbytehq/airbyte/pull/14437) | 🎉 Source Dixa: bump version 0.1.3 | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367) | Migrated to CI Sandbox, refactorred code structure for future support | From 521a798fb5361b4d341641a0e40c221715e2cbd3 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Mon, 12 Jun 2023 14:59:57 -0700 Subject: [PATCH 02/18] Experiments to fix CI (#27262) * don't pin virtualenv version * comments * relax python version * keep virtualenv relax * remove cache * add note * Update .github/workflows/gradle.yml Co-authored-by: Ben Church --------- Co-authored-by: Ben Church --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index f4222cca7cc0a..e27b3ffa870a3 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -254,7 +254,7 @@ jobs: - name: Cache Build Artifacts uses: ./.github/actions/cache-build-artifacts with: - cache-key: ${{ secrets.CACHE_VERSION }}-connectors + cache-key: ${{ secrets.CACHE_VERSION }}-connectors-gradle-build - uses: actions/setup-java@v3 with: @@ -270,7 +270,7 @@ jobs: python-version: "3.9" - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user + run: python3 -m pip install virtualenv --user - name: Install automake run: apt-get update && apt-get install -y automake build-essential libtool libtool-bin autoconf From f8b9ee228969d467941153b9f7fc8d845b77c124 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 12 Jun 2023 15:54:10 -0700 Subject: [PATCH 03/18] Metadata: Add Breaking Changes entry (#27178) * Add breaking changes metadata * Add valid breaking change * Add invalid cases * Update comment * Add invalid deadline check * Convert to releases * Update to releases * Test for migration doc url * Add registry override application * Add even more tests * Add default tests * Apply format * Add releases to registry model * Add new test cases * Set additional properties to false --------- Co-authored-by: Octavia Squidington III --- .../ConnectorMetadataDefinitionV0.py | 43 +++- .../ConnectorRegistryDestinationDefinition.py | 41 +++- .../ConnectorRegistrySourceDefinition.py | 41 +++- .../models/generated/ConnectorRegistryV0.py | 42 +++- .../models/generated/ConnectorReleases.py | 47 +++++ .../models/generated/__init__.py | 1 + .../src/ConnectorMetadataDefinitionV0.yaml | 2 + ...onnectorRegistryDestinationDefinition.yaml | 2 + .../ConnectorRegistrySourceDefinition.yaml | 2 + .../models/src/ConnectorReleases.yaml | 44 ++++ ...d_breaking_change_additional_property.yaml | 20 ++ ...alid_breaking_change_invalid_deadline.yaml | 19 ++ ...a_invalid_breaking_change_no_deadline.yaml | 18 ++ ...ta_invalid_breaking_change_no_message.yaml | 18 ++ ...adata_invalid_breaking_change_version.yaml | 19 ++ .../valid/metadata_breaking_changes.yaml | 19 ++ ...eaking_changes_with_migration_doc_url.yaml | 21 ++ .../orchestrator/assets/registry.py | 48 ++++- .../orchestrator/tests/test_registry.py | 194 +++++++++++++++++- 19 files changed, 629 insertions(+), 12 deletions(-) create mode 100644 airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py create mode 100644 airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml create mode 100644 airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py index f7151f34915cf..44c4666bfd27d 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py @@ -4,10 +4,10 @@ from __future__ import annotations from datetime import date -from typing import List, Optional +from typing import Dict, List, Optional from uuid import UUID -from pydantic import AnyUrl, BaseModel, Extra, Field +from pydantic import AnyUrl, BaseModel, Extra, Field, constr from typing_extensions import Literal @@ -75,6 +75,23 @@ class JobType(BaseModel): ) +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version}", + ) + + class JobTypeResourceLimit(BaseModel): class Config: extra = Extra.forbid @@ -83,6 +100,16 @@ class Config: resourceRequirements: ResourceRequirements +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ActorDefinitionResourceRequirements(BaseModel): class Config: extra = Extra.forbid @@ -94,6 +121,17 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide", + ) + + class RegistryOverrides(BaseModel): class Config: extra = Extra.forbid @@ -154,6 +192,7 @@ class Data(BaseModel): ) registries: Optional[Registry] = None allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorReleases] = None normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None suggestedStreams: Optional[SuggestedStreams] = None resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py index dce84d5b13fb9..cbfdfecd2cf62 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import BaseModel, Extra, Field +from pydantic import AnyUrl, BaseModel, Extra, Field, constr from typing_extensions import Literal @@ -73,6 +73,23 @@ class Config: ) +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version}", + ) + + class JobTypeResourceLimit(BaseModel): class Config: extra = Extra.forbid @@ -81,6 +98,16 @@ class Config: resourceRequirements: ResourceRequirements +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ActorDefinitionResourceRequirements(BaseModel): class Config: extra = Extra.forbid @@ -92,6 +119,17 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide", + ) + + class ConnectorRegistryDestinationDefinition(BaseModel): class Config: extra = Extra.allow @@ -134,3 +172,4 @@ class Config: description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", ) allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorReleases] = None diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py index 257fcdda0ef28..e37c027bfc0a6 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import BaseModel, Extra, Field +from pydantic import AnyUrl, BaseModel, Extra, Field, constr from typing_extensions import Literal @@ -65,6 +65,23 @@ class Config: ) +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version}", + ) + + class JobTypeResourceLimit(BaseModel): class Config: extra = Extra.forbid @@ -73,6 +90,16 @@ class Config: resourceRequirements: ResourceRequirements +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ActorDefinitionResourceRequirements(BaseModel): class Config: extra = Extra.forbid @@ -84,6 +111,17 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide", + ) + + class ConnectorRegistrySourceDefinition(BaseModel): class Config: extra = Extra.allow @@ -123,3 +161,4 @@ class Config: None, description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", ) + releases: Optional[ConnectorReleases] = None diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py index ddab4df78eb03..027b9ac59f1d0 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import BaseModel, Extra, Field +from pydantic import AnyUrl, BaseModel, Extra, Field, constr from typing_extensions import Literal @@ -73,6 +73,23 @@ class Config: ) +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version}", + ) + + class SuggestedStreams(BaseModel): class Config: extra = Extra.allow @@ -91,6 +108,16 @@ class Config: resourceRequirements: ResourceRequirements +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ActorDefinitionResourceRequirements(BaseModel): class Config: extra = Extra.forbid @@ -102,6 +129,17 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide", + ) + + class ConnectorRegistrySourceDefinition(BaseModel): class Config: extra = Extra.allow @@ -141,6 +179,7 @@ class Config: None, description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", ) + releases: Optional[ConnectorReleases] = None class ConnectorRegistryDestinationDefinition(BaseModel): @@ -185,6 +224,7 @@ class Config: description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", ) allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorReleases] = None class ConnectorRegistryV0(BaseModel): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py new file mode 100644 index 0000000000000..7cf3b2c45580f --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py @@ -0,0 +1,47 @@ +# generated by datamodel-codegen: +# filename: ConnectorReleases.yaml + +from __future__ import annotations + +from datetime import date +from typing import Dict, Optional + +from pydantic import AnyUrl, BaseModel, Extra, Field, constr + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version}", + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide", + ) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py index 1091c7cf5fcf4..88407c9531840 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py @@ -5,6 +5,7 @@ from .ConnectorRegistryDestinationDefinition import * from .ConnectorRegistrySourceDefinition import * from .ConnectorRegistryV0 import * +from .ConnectorReleases import * from .JobType import * from .NormalizationDestinationDefinitionConfig import * from .RegistryOverrides import * diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml index 691b7f22829ef..16d9a60613601 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml @@ -98,6 +98,8 @@ properties: allowedHosts: "$ref": AllowedHosts.yaml + releases: + "$ref": ConnectorReleases.yaml normalizationConfig: "$ref": NormalizationDestinationDefinitionConfig.yaml suggestedStreams: diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml index fa8c762b73361..35f266a56c415 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml @@ -66,3 +66,5 @@ properties: description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used. allowedHosts: "$ref": AllowedHosts.yaml + releases: + "$ref": ConnectorReleases.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml index 338475eff0be8..9b1640af0bb4a 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml @@ -68,3 +68,5 @@ properties: maxSecondsBetweenMessages: description: Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach type: integer + releases: + "$ref": ConnectorReleases.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml new file mode 100644 index 0000000000000..3a8797886f944 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml @@ -0,0 +1,44 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml +title: ConnectorReleases +description: Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade. +type: object +additionalProperties: false +required: + - breakingChanges +properties: + breakingChanges: + $ref: "#/definitions/ConnectorBreakingChanges" + migrationDocumentationUrl: + description: URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}/migration_guide + type: string + format: uri +definitions: + ConnectorBreakingChanges: + description: Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade. + type: object + additionalProperties: false + minProperties: 1 + patternProperties: + "^\\d+\\.\\d+\\.\\d+$": + $ref: "#/definitions/VersionBreakingChange" + VersionBreakingChange: + description: Contains information about a breaking change, including the deadline to upgrade and a message detailing the change. + type: object + additionalProperties: false + required: + - upgradeDeadline + - message + properties: + upgradeDeadline: + description: The deadline by which to upgrade before the breaking change takes effect. + type: string + format: date + message: + description: Descriptive message detailing the breaking change. + type: string + migrationDocumentationUrl: + description: URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}/migration_guide#${version} + type: string + format: uri diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml new file mode 100644 index 0000000000000..d60a7738a88b1 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml @@ -0,0 +1,20 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + addition: "hi" + 2.1.3: + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + upgradeDeadline: 2023-08-22 + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml new file mode 100644 index 0000000000000..e495455517cb4 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml @@ -0,0 +1,19 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 1.2.3: + upgradeDeadline: 2023-08-22-11 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml new file mode 100644 index 0000000000000..e7c1a974ec13c --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 1.2.3: + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml new file mode 100644 index 0000000000000..96d3db77c8535 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.1.3: + upgradeDeadline: 2023-08-22 + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml new file mode 100644 index 0000000000000..645ffa4fac7aa --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml @@ -0,0 +1,19 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + v2.1.3: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml new file mode 100644 index 0000000000000..b851aa1f82f21 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml @@ -0,0 +1,19 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml new file mode 100644 index 0000000000000..ddfde0035bd21 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml @@ -0,0 +1,21 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1-exists + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/alloydb/migration_guide + breakingChanges: + 2.0.0: + migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/alloydb/migration_guide#2.0.0 + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py index 4c08a28e30713..70608b43b6eaa 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py @@ -3,7 +3,7 @@ # import copy import json -from typing import List +from typing import List, Optional from pydash.objects import get import pandas as pd @@ -54,6 +54,44 @@ def apply_overrides_from_registry(metadata_data: dict, override_registry_key: st return metadata_data +def calculate_migration_documentation_url(releases_or_breaking_change: dict, documentation_url: str, version: Optional[str] = None) -> str: + """Calculate the migration documentation url for the connector releases. + + Args: + metadata_releases (dict): The connector releases. + + Returns: + str: The migration documentation url. + """ + + base_url = f"{documentation_url}/migration_guide" + default_migration_documentation_url = f"{base_url}#{version}" if version is not None else base_url + + return releases_or_breaking_change.get("migrationDocumentationUrl", default_migration_documentation_url) + + +@deep_copy_params +def apply_connector_release_defaults(metadata: dict) -> Optional[pd.DataFrame]: + metadata_releases = metadata.get("releases") + documentation_url = metadata.get("documentationUrl") + if metadata_releases is None: + return None + + # apply defaults for connector releases + metadata_releases["migrationDocumentationUrl"] = calculate_migration_documentation_url(metadata_releases, documentation_url) + + # releases has a dictionary field called breakingChanges, where the key is the version and the value is the data for the breaking change + # each breaking change has a migrationDocumentationUrl field that is optional, so we need to apply defaults to it + breaking_changes = metadata_releases["breakingChanges"] + if breaking_changes is not None: + for version, breaking_change in breaking_changes.items(): + breaking_change["migrationDocumentationUrl"] = calculate_migration_documentation_url( + breaking_change, documentation_url, version + ) + + return metadata_releases + + @deep_copy_params def metadata_to_registry_entry(metadata_entry: LatestMetadataEntry, connector_type: str, override_registry_key: str) -> dict: """Convert the metadata definition to a registry entry. @@ -70,9 +108,11 @@ def metadata_to_registry_entry(metadata_entry: LatestMetadataEntry, connector_ty metadata_data = metadata_definition["data"] + # apply overrides from the registry overrode_metadata_data = apply_overrides_from_registry(metadata_data, override_registry_key) - del overrode_metadata_data["registries"] + # remove fields that are not needed in the registry + del overrode_metadata_data["registries"] del overrode_metadata_data["connectorType"] # rename field connectorSubtype to sourceType @@ -92,12 +132,12 @@ def metadata_to_registry_entry(metadata_entry: LatestMetadataEntry, connector_ty overrode_metadata_data["public"] = True # if there is no releaseStage, set it to "alpha" - # Note: this is something our current cloud registry generator does - # Note: We will not once this is live if not overrode_metadata_data.get("releaseStage"): overrode_metadata_data["releaseStage"] = "alpha" + # apply generated fields overrode_metadata_data["iconUrl"] = metadata_entry.icon_url + overrode_metadata_data["releases"] = apply_connector_release_defaults(overrode_metadata_data) return overrode_metadata_data diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py index 89ab004c18044..15ee06458c6c9 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py @@ -1,11 +1,11 @@ +import pytest +from unittest.mock import Mock from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 - +from orchestrator.assets.registry import metadata_to_registry_entry from orchestrator.assets.registry_report import ( all_sources_dataframe, all_destinations_dataframe, -) -from orchestrator.assets.registry_report import ( oss_destinations_dataframe, cloud_destinations_dataframe, oss_sources_dataframe, @@ -52,3 +52,191 @@ def test_merged_registry_dataframes(oss_registry_dict, cloud_registry_dict): all_destination_definition_ids = set(all_destinations_df["definitionId"]) assert all_destination_definition_ids == oss_destination_definition_ids.union(cloud_destination_definition_ids) + + +@pytest.mark.parametrize( + "registry_type, connector_type, expected_id_field", + [ + ("cloud", "source", "sourceDefinitionId"), + ("cloud", "destination", "destinationDefinitionId"), + ("oss", "source", "sourceDefinitionId"), + ("oss", "destination", "destinationDefinitionId"), + ], +) +def test_definition_id_conversion(registry_type, connector_type, expected_id_field): + """ + Test if the definitionId in the metadata is successfully converted to + destinationDefinitionId or sourceDefinitionId in the registry entry. + """ + metadata = {"data": {"connectorType": connector_type, "definitionId": "test-id", "registries": {registry_type: {"enabled": True}}}} + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, connector_type, registry_type) + assert "definitionId" not in result + assert result[expected_id_field] == "test-id" + + +def test_tombstone_custom_public_set(): + """ + Test if tombstone, custom and public are set correctly in the registry entry. + """ + metadata = {"data": {"connectorType": "source", "definitionId": "test-id", "registries": {"oss": {"enabled": True}}}} + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["tombstone"] is False + assert result["custom"] is False + assert result["public"] is True + + +def test_fields_deletion(): + """ + Test if registries, connectorType, and definitionId fields were deleted from the registry entry. + """ + metadata = {"data": {"connectorType": "source", "definitionId": "test-id", "registries": {"oss": {"enabled": True}}}} + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert "registries" not in result + assert "connectorType" not in result + assert "definitionId" not in result + + +@pytest.mark.parametrize( + "registry_type, expected_docker_image_tag, expected_additional_field", + [("cloud", "cloud_tag", "cloud_value"), ("oss", "oss_tag", "oss_value")], +) +def test_overrides_application(registry_type, expected_docker_image_tag, expected_additional_field): + """ + Test if the overrides for cloud or oss are properly applied to the registry entry. + """ + # Assuming 'overriddenField' is a field to be overridden in metadata for the sake of this test. + metadata = { + "data": { + "connectorType": "source", + "definitionId": "test-id", + "dockerImageTag": "base_tag", + "registries": { + "oss": {"enabled": True, "dockerImageTag": "oss_tag", "additionalField": "oss_value"}, + "cloud": {"enabled": True, "dockerImageTag": "cloud_tag", "additionalField": "cloud_value"}, + }, + } + } + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", registry_type) + assert result["dockerImageTag"] == expected_docker_image_tag + assert result["additionalField"] == expected_additional_field + + +def test_source_type_extraction(): + """ + Test if sourceType is successfully extracted from connectorSubtype in the registry entry. + """ + metadata = { + "data": { + "connectorType": "source", + "connectorSubtype": "database", + "definitionId": "test-id", + "registries": {"oss": {"enabled": True}}, + } + } + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["sourceType"] == "database" + + +def test_release_stage_default(): + """ + Test if releaseStage is defaulted to alpha in the registry entry. + """ + metadata = {"data": {"connectorType": "source", "definitionId": "test-id", "registries": {"oss": {"enabled": True}}}} + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["releaseStage"] == "alpha" + + +def test_migration_documentation_url_default(): + """ + Test if migrationDocumentationUrl is successfully defaulted in releases.migrationDocumentationUrl in the registry entry. + """ + metadata = { + "data": { + "connectorType": "source", + "definitionId": "test-id", + "documentationUrl": "test-doc-url", + "registries": {"oss": {"enabled": True}}, + "releases": {"breakingChanges": {"1.0.0": {}}}, + } + } + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + expected_top_migration_documentation_url = "test-doc-url/migration_guide" + expected_version_migration_documentation_url = "test-doc-url/migration_guide#1.0.0" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["releases"]["migrationDocumentationUrl"] == expected_top_migration_documentation_url + assert result["releases"]["breakingChanges"]["1.0.0"]["migrationDocumentationUrl"] == expected_version_migration_documentation_url + + +def test_breaking_changes_migration_documentation_url(): + """ + Test if migrationDocumentationUrl is successfully defaulted for all entries in releases.breakingChanges including the key as the version. + """ + metadata = { + "data": { + "connectorType": "source", + "definitionId": "test-id", + "documentationUrl": "test-doc-url", + "registries": {"oss": {"enabled": True}}, + "releases": { + "migrationDocumentationUrl": "test-migration-doc-url", + "breakingChanges": {"1.0.0": {"migrationDocumentationUrl": "test-migration-doc-url-version"}}, + }, + } + } + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["releases"]["migrationDocumentationUrl"] == "test-migration-doc-url" + assert result["releases"]["breakingChanges"]["1.0.0"]["migrationDocumentationUrl"] == "test-migration-doc-url-version" + + +def test_icon_url(): + """ + Test if the iconUrl in the metadata entry is correctly set in the registry entry. + """ + metadata = {"data": {"connectorType": "source", "definitionId": "test-id", "registries": {"oss": {"enabled": True}}}} + + mock_metadata_entry = Mock() + mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.icon_url = "test-icon-url" + + result = metadata_to_registry_entry(mock_metadata_entry, "source", "oss") + assert result["iconUrl"] == "test-icon-url" From 27635ba26ae2d9a1ac4d34f20b13ff61fae91ef8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 13 Jun 2023 02:39:12 +0200 Subject: [PATCH 04/18] Low code CDK: Datetime format documentation (#27149) * add format documentation * fix * improve --- .../declarative_component_schema.yaml | 64 ++++++++++++++++++- .../models/declarative_component_schema.py | 8 +-- .../connector-builder-ui/incremental-sync.md | 2 +- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index ca81261e5e4f3..d628f04d7fb0f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -580,10 +580,40 @@ definitions: - "{{ config['record_cursor'] }}" datetime_format: title: Cursor Field Datetime Format - description: The datetime format of the Cursor Field. + description: | + The datetime format of the Cursor Field. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available: + * **%s**: Epoch unix timestamp - `1686218963` + * **%a**: Weekday (abbreviated) - `Sun` + * **%A**: Weekday (full) - `Sunday` + * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday) + * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31` + * **%b**: Month (abbreviated) - `Jan` + * **%B**: Month (full) - `January` + * **%m**: Month (zero-padded) - `01`, `02`, ..., `12` + * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99` + * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999` + * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23` + * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12` + * **%p**: AM/PM indicator + * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59` + * **%S**: Second (zero-padded) - `00`, `01`, ..., `59` + * **%f**: Microsecond (zero-padded to 6 digits) - `000000` + * **%z**: UTC offset - `(empty)`, `+0000`, `-0400` + * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT` + * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366` + * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53` + * **%W**: Week number of the year (starting Monday) - `00`, ..., `53` + * **%c**: Date and time - `Tue Aug 16 21:30:00 1988` + * **%x**: Date standard format - `08/16/1988` + * **%X**: Time standard format - `21:30:00` + * **%%**: Literal '%' character + + Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes). type: string examples: - "%Y-%m-%dT%H:%M:%S.%f%z" + - "%Y-%m-%d" + - "%s" cursor_granularity: title: Cursor Granularity description: @@ -1283,11 +1313,39 @@ definitions: - "{{ config['start_time'] }}" datetime_format: title: Datetime Format - description: Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp). + description: | + Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available: + * **%s**: Epoch unix timestamp - `1686218963` + * **%a**: Weekday (abbreviated) - `Sun` + * **%A**: Weekday (full) - `Sunday` + * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday) + * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31` + * **%b**: Month (abbreviated) - `Jan` + * **%B**: Month (full) - `January` + * **%m**: Month (zero-padded) - `01`, `02`, ..., `12` + * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99` + * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999` + * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23` + * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12` + * **%p**: AM/PM indicator + * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59` + * **%S**: Second (zero-padded) - `00`, `01`, ..., `59` + * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999` + * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216` + * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT` + * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366` + * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53` + * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53` + * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988` + * **%x**: Date representation - `08/16/1988` + * **%X**: Time representation - `21:30:00` + * **%%**: Literal '%' character + + Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes). type: string default: "" examples: - - "%Y-%m-%dT%H:%M:%S.%f%" + - "%Y-%m-%dT%H:%M:%S.%f%z" - "%Y-%m-%d" - "%s" max_datetime: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 0f2e2805b1410..5abebc04a53f3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -453,8 +453,8 @@ class MinMaxDatetime(BaseModel): ) datetime_format: Optional[str] = Field( "", - description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).', - examples=["%Y-%m-%dT%H:%M:%S.%f%", "%Y-%m-%d", "%s"], + description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], title="Datetime Format", ) max_datetime: Optional[str] = Field( @@ -806,8 +806,8 @@ class DatetimeBasedCursor(BaseModel): ) datetime_format: str = Field( ..., - description="The datetime format of the Cursor Field.", - examples=["%Y-%m-%dT%H:%M:%S.%f%z"], + description="The datetime format of the Cursor Field. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], title="Cursor Field Datetime Format", ) cursor_granularity: Optional[str] = Field( diff --git a/docs/connector-development/connector-builder-ui/incremental-sync.md b/docs/connector-development/connector-builder-ui/incremental-sync.md index 8eb7c621eec2c..7a4b751bada03 100644 --- a/docs/connector-development/connector-builder-ui/incremental-sync.md +++ b/docs/connector-development/connector-builder-ui/incremental-sync.md @@ -20,7 +20,7 @@ To configure incremental syncs for a stream in the connector builder, you have t In the builder UI, these things are specified like this: * The "Cursor field" is the property in the record that defines the date and time when the record got changed. It's used to decide which records are synced already and which records are "new" -* The "Datetime format" specifies the [format](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) the cursor field is using to specify date and time, +* The "Datetime format" specifies the format the cursor field is using to specify date and time. Check out the [YAML reference](/connector-development/config-based/understanding-the-yaml-file/reference#/definitions/DatetimeBasedCursor) for a full list of supported formats. * The "Cursor granularity" is the smallest time unit supported by the API to specify the time range to request records for expressed as [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) * The "Start datetime" is the initial start date of the time range to fetch records for. When doing incremental syncs, the second sync will overwrite this date with the last record that got synced so far. * The "End datetime" is the end date of the time range to fetch records for. In most cases it's set to the current date and time when the sync is started to sync all changes that happened so far. From 769ed3efa70dbb5674b190974f81e0c79a541afd Mon Sep 17 00:00:00 2001 From: Ryan Fu Date: Mon, 12 Jun 2023 18:27:13 -0700 Subject: [PATCH 05/18] Adds PartialAirbyteMessage overhead & removes serialize/deserialize on CSV writer (#27222) * No Op change to see if Connectors Base is unrelated * Isolated test to see the memory used after processing CSV writer * Changes record writer * Removed Json.serialize on string * Turn on Snowflake Async * Fixed emittedAt timestamp in milliseconds * Add comments. Undo Evan build changes. * Allow remote debugging. * Adds calc for PartialAirbyteRecordMessage memory overhead * Revert back airbyte build architecture * Passes in the parsed emittedAt to the CSVwriter * Cleans up lingering comments * Performance enhancement - increase the available JVM memory known by the container * Serializes the data earlier to avoid normalization issues and lowers optimal batch size * Passes messageString instead of serializing the data due to memory overhead causes OOM early * Removes memory overhead of record data * Cleans up PR, fixes memory issues & increases throughput by lowering optimal batch size * Add calculation comment & removes procps from docker install * Automated Commit - Format and Process Resources Changes * Turns on Async for testing purposes * Moves up implementation higher up the classes to remove unnecessary no-op * Automated Commit - Format and Process Resources Changes * Adds TODOs for migrating buffers to use only serialized records * Breaks down calc & throws UnsupportedOperationException * Adds TODO for migration all destinations to use the getDataRow(id, formattedString, emittedAt) to avoid unnecessary ser-de overhead --------- Co-authored-by: Davin Chia Co-authored-by: ryankfu --- .../s3/csv/BaseSheetGenerator.java | 6 +++- .../s3/csv/CsvSerializedBuffer.java | 12 +++++++ .../destination/s3/csv/CsvSheetGenerator.java | 4 +++ .../RootLevelFlatteningSheetGenerator.java | 2 +- .../csv/StagingDatabaseCsvSheetGenerator.java | 8 +++++ .../s3/jsonl/JsonLSerializedBuffer.java | 2 +- .../s3/parquet/ParquetSerializedBuffer.java | 9 +++-- .../record_buffer/BaseSerializedBuffer.java | 36 ++++++++++++++++++- .../record_buffer/SerializableBuffer.java | 12 +++++++ .../AsyncStreamConsumer.java | 8 ++++- .../PartialAirbyteRecordMessage.java | 27 ++++++++++++-- .../destination/staging/AsyncFlush.java | 5 ++- .../destination-snowflake/build.gradle | 4 +-- ...SnowflakeInternalStagingSqlOperations.java | 6 ++-- 14 files changed, 124 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java index 0ed30bf2fb159..e813f1ad52ae5 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java @@ -25,10 +25,14 @@ public List getDataRow(final UUID id, final AirbyteRecordMessage recordM } @Override - public List getDataRow(JsonNode formattedData) { + public List getDataRow(final JsonNode formattedData) { return new LinkedList<>(getRecordColumns(formattedData)); } + public List getDataRow(final UUID id, final String formattedString, final long emittedAt) { + throw new UnsupportedOperationException("Not implemented in BaseSheetGenerator"); + } + abstract List getRecordColumns(JsonNode json); } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java index 9cb2b2a4f7190..b535e20660f14 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java @@ -59,11 +59,23 @@ protected void initWriter(final OutputStream outputStream) throws IOException { csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), csvFormat); } + /** + * TODO: (ryankfu) remove this call within {@link SerializedBufferingStrategy} and move to use + * recordString + * + * @param record AirbyteRecordMessage to be written + * @throws IOException + */ @Override protected void writeRecord(final AirbyteRecordMessage record) throws IOException { csvPrinter.printRecord(csvSheetGenerator.getDataRow(UUID.randomUUID(), record)); } + @Override + protected void writeRecord(final String recordString, final long emittedAt) throws IOException { + csvPrinter.printRecord(csvSheetGenerator.getDataRow(UUID.randomUUID(), recordString, emittedAt)); + } + @Override protected void flushWriter() throws IOException { // in an async world, it is possible that flush writer gets called even if no records were accepted. diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java index 4c174eb803134..72512c26a7179 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java @@ -18,10 +18,14 @@ public interface CsvSheetGenerator { List getHeaderRow(); + // TODO: (ryankfu) remove this and switch over all destinations to pass in serialized recordStrings, + // both for performance and lowers memory footprint List getDataRow(UUID id, AirbyteRecordMessage recordMessage); List getDataRow(JsonNode formattedData); + List getDataRow(UUID id, String formattedString, long emittedAt); + final class Factory { public static CsvSheetGenerator create(final JsonNode jsonSchema, final S3CsvFormatConfig formatConfig) { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java index 8fddaf2a21790..2d86f26d55f31 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java @@ -22,7 +22,7 @@ public class RootLevelFlatteningSheetGenerator extends BaseSheetGenerator implem public RootLevelFlatteningSheetGenerator(final JsonNode jsonSchema) { this.recordHeaders = MoreIterators.toList(jsonSchema.get("properties").fieldNames()) - .stream().sorted().collect(Collectors.toList());; + .stream().sorted().collect(Collectors.toList()); } @Override diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java index 8dbfc4bedc7e8..1678a1c5b9cc2 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java @@ -49,4 +49,12 @@ public List getDataRow(final JsonNode formattedData) { return new LinkedList<>(Collections.singletonList(Jsons.serialize(formattedData))); } + @Override + public List getDataRow(final UUID id, final String formattedString, final long emittedAt) { + return List.of( + id, + formattedString, + Timestamp.from(Instant.ofEpochMilli(emittedAt))); + } + } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java index 8aa8a9d78b8d4..b2e05d704abdf 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java @@ -53,7 +53,7 @@ protected void writeRecord(final AirbyteRecordMessage record) { json.put(JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString()); json.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, record.getEmittedAt()); if (flattenData) { - Map data = MAPPER.convertValue(record.getData(), new TypeReference<>() {}); + final Map data = MAPPER.convertValue(record.getData(), new TypeReference<>() {}); json.setAll(data); } else { json.set(JavaBaseConstants.COLUMN_NAME_DATA, record.getData()); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java index 746c279fc9ead..67896c2de35c2 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java @@ -38,7 +38,7 @@ * The {@link io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer} class * abstracts the {@link io.airbyte.integrations.destination.record_buffer.BufferStorage} from the * details of the format the data is going to be stored in. - * + *

* Unfortunately, the Parquet library doesn't allow us to manipulate the output stream and forces us * to go through {@link HadoopOutputFile} instead. So we can't benefit from the abstraction * described above. Therefore, we re-implement the necessary methods to be used as @@ -72,7 +72,7 @@ public ParquetSerializedBuffer(final S3DestinationConfig config, Files.deleteIfExists(bufferFile); avroRecordFactory = new AvroRecordFactory(schema, AvroConstants.JSON_CONVERTER); final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); - Configuration avroConfig = new Configuration(); + final Configuration avroConfig = new Configuration(); avroConfig.setBoolean(WRITE_OLD_LIST_STRUCTURE, false); parquetWriter = AvroParquetWriter.builder(HadoopOutputFile .fromPath(new org.apache.hadoop.fs.Path(bufferFile.toUri()), avroConfig)) @@ -101,6 +101,11 @@ public long accept(final AirbyteRecordMessage record) throws Exception { } } + @Override + public long accept(final String recordString, final long emittedAt) throws Exception { + throw new UnsupportedOperationException("This method is not supported for ParquetSerializedBuffer"); + } + @Override public void flush() throws Exception { if (inputStream == null && !isClosed) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java index 73d16162aa3fa..39b2925f9ee11 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.record_buffer; import com.google.common.io.CountingOutputStream; +import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.IOException; @@ -18,7 +19,7 @@ /** * Base implementation of a {@link SerializableBuffer}. It is composed of a {@link BufferStorage} * where the actual data is being stored in a serialized format. - * + *

* Such data format is defined by concrete implementation inheriting from this base abstract class. * To do so, necessary methods on handling "writer" methods should be defined. This writer would * take care of converting {@link AirbyteRecordMessage} into the serialized form of the data such as @@ -57,8 +58,21 @@ protected BaseSerializedBuffer(final BufferStorage bufferStorage) throws Excepti * Transform the @param record into a serialized form of the data and writes it to the registered * OutputStream provided when {@link BaseSerializedBuffer#initWriter} was called. */ + @Deprecated protected abstract void writeRecord(AirbyteRecordMessage record) throws IOException; + /** + * TODO: (ryankfu) move destination to use serialized record string instead of passing entire + * AirbyteRecord + * + * @param recordString serialized record + * @param emittedAt timestamp of the record in milliseconds + * @throws IOException + */ + protected void writeRecord(final String recordString, final long emittedAt) throws IOException { + writeRecord(Jsons.deserialize(recordString, AirbyteRecordMessage.class).withEmittedAt(emittedAt)); + } + /** * Stops the writer from receiving new data and prepares it for being finalized and converted into * an InputStream to read from instead. This is used when flushing the buffer into some other @@ -96,6 +110,26 @@ public long accept(final AirbyteRecordMessage record) throws Exception { } } + @Override + public long accept(final String recordString, final long emittedAt) throws Exception { + if (!isStarted) { + if (useCompression) { + compressedBuffer = new GzipCompressorOutputStream(byteCounter); + initWriter(compressedBuffer); + } else { + initWriter(byteCounter); + } + isStarted = true; + } + if (inputStream == null && !isClosed) { + final long startCount = byteCounter.getCount(); + writeRecord(recordString, emittedAt); + return byteCounter.getCount() - startCount; + } else { + throw new IllegalCallerException("Buffer is already closed, it cannot accept more messages"); + } + } + @Override public String getFilename() throws IOException { if (useCompression && !bufferStorage.getFilename().endsWith(GZ_SUFFIX)) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java index 2762ab055ecb9..1870d779b70fd 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java @@ -35,8 +35,20 @@ public interface SerializableBuffer extends AutoCloseable { * @param record {@link AirbyteRecordMessage} to be added to buffer * @return number of bytes written to the buffer */ + @Deprecated long accept(AirbyteRecordMessage record) throws Exception; + /** + * TODO: (ryankfu) Move all destination connectors to pass the serialized record string instead of + * the entire AirbyteRecordMessage + * + * @param recordString serialized record + * @param emittedAt timestamp of the record in milliseconds + * @return number of bytes written to the buffer + * @throws Exception + */ + long accept(String recordString, long emittedAt) throws Exception; + /** * Flush a buffer implementation. */ diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java index 600352e2759bc..04ba55d54dadd 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java @@ -50,6 +50,12 @@ public class AsyncStreamConsumer implements SerializedAirbyteMessageConsumer { private boolean hasStarted; private boolean hasClosed; + // This is to account for the references when deserialization to a PartialAirbyteMessage. The + // calculation is as follows: + // PartialAirbyteMessage (4) + Max( PartialRecordMessage(4), PartialStateMessage(6)) with + // PartialStateMessage being larger with more nested objects within it. Using 8 bytes as we assumed + // a 64 bit JVM. + final int PARTIAL_DESERIALIZE_REF_BYTES = 10 * 8; public AsyncStreamConsumer(final Consumer outputRecordCollector, final OnStartFunction onStart, @@ -107,7 +113,7 @@ public void accept(final String messageString, final Integer sizeInBytes) throws validateRecord(message); } - bufferEnqueue.addRecord(message, sizeInBytes); + bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES); }); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java index 9fb7e5ef9a2d4..70088554784eb 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import java.util.Objects; public class PartialAirbyteRecordMessage { @@ -14,6 +15,10 @@ public class PartialAirbyteRecordMessage { @JsonProperty("stream") private String stream; + @JsonProperty("emitted_at") + @JsonPropertyDescription("when the data was emitted from the source. epoch in millisecond.") + private long emittedAt; + public PartialAirbyteRecordMessage() {} @JsonProperty("namespace") @@ -46,6 +51,21 @@ public PartialAirbyteRecordMessage withStream(final String stream) { return this; } + @JsonProperty("emitted_at") + public Long getEmittedAt() { + return this.emittedAt; + } + + @JsonProperty("emitted_at") + public void setEmittedAt(final long emittedAt) { + this.emittedAt = emittedAt; + } + + public PartialAirbyteRecordMessage withEmittedAt(final Long emittedAt) { + this.emittedAt = emittedAt; + return this; + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -55,12 +75,14 @@ public boolean equals(final Object o) { return false; } final PartialAirbyteRecordMessage that = (PartialAirbyteRecordMessage) o; - return Objects.equals(namespace, that.namespace) && Objects.equals(stream, that.stream); + return Objects.equals(namespace, that.namespace) + && Objects.equals(stream, that.stream) + && Objects.equals(emittedAt, that.emittedAt); } @Override public int hashCode() { - return Objects.hash(namespace, stream); + return Objects.hash(namespace, stream, emittedAt); } @Override @@ -68,6 +90,7 @@ public String toString() { return "PartialAirbyteRecordMessage{" + "namespace='" + namespace + '\'' + ", stream='" + stream + '\'' + + ", emittedAt='" + emittedAt + '\'' + '}'; } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java index c6a25fb43abab..d070ad17f79bc 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java @@ -12,7 +12,6 @@ import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; import io.airbyte.integrations.destination_async.DestinationFlushFunction; import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.List; @@ -57,7 +56,7 @@ public void flush(final StreamDescriptor decs, final Stream e); } } @@ -193,7 +193,7 @@ public void dropStageIfExists(final JdbcDatabase database, final String stageNam final String query = getDropQuery(stageName); LOGGER.debug("Executing query: {}", query); database.execute(query); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } @@ -214,7 +214,7 @@ public void cleanUpStage(final JdbcDatabase database, final String stageName, fi final String query = getRemoveQuery(stageName); LOGGER.debug("Executing query: {}", query); database.execute(query); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } From a816d392235a446dbe39a902676af4fc29a262f0 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 12 Jun 2023 18:51:28 -0700 Subject: [PATCH 06/18] Add icon docs (#27295) Co-authored-by: Octavia Squidington III --- airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml | 1 + .../connector-metadata-file.md | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml b/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml index 61db82fc08e8b..5f60b0eb11527 100644 --- a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml +++ b/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml @@ -8,3 +8,4 @@ paths: - Changelog updated in `docs/integrations//.md` with an entry for the new version. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog) - You, or an Airbyter, have run `/test` successfully on this PR - or on a non-forked branch - You've updated the connector's `metadata.yaml` file (new!) + - If set, you've ensured the icon is present in the `platform-internal` repo. ([Docs](https://docs.airbyte.com/connector-development/connector-metadata-file/#the-icon-field)) diff --git a/docs/connector-development/connector-metadata-file.md b/docs/connector-development/connector-metadata-file.md index 7a274de5dba02..8bb8d67979d1d 100644 --- a/docs/connector-development/connector-metadata-file.md +++ b/docs/connector-development/connector-metadata-file.md @@ -85,4 +85,16 @@ In the example above, the connector has three tags. Tags are used for two primar 2. **Keywords for Searching**: Tags that begin with keyword: are used to make the connector more discoverable by adding searchable terms related to it. In the example above, the tags keyword:database and keyword:SQL can be used to find this connector when searching for `database` or `SQL`. -These are just examples of how tags can be used. As a free-form field, the tags list can be customized as required for each connector. This flexibility allows tags to be a powerful tool for managing and discovering connectors. \ No newline at end of file +These are just examples of how tags can be used. As a free-form field, the tags list can be customized as required for each connector. This flexibility allows tags to be a powerful tool for managing and discovering connectors. + +## The `icon` Field +This denotes the name of the icon file for the connector. At this time the icon file is located in the `platform-internal` repository. So please ensure that the icon file is present in the `platform-internal` repository at [oss/airbyte-config/init/src/main/resources/icons](https://github.com/airbytehq/airbyte-platform-internal/tree/master/oss/airbyte-config/init/src/main/resources/icons) before adding the icon name to the `metadata.yaml` file. + +### Future Plans +_⚠️ This property is in the process of being refactored to be a file in the connector folder_ + +You may notice a `icon.svg` file in the connectors folder. + +This is because we are transitioning away from icons being stored in the `platform-internal` repository. Instead, we will be storing them in the connector folder itself. This will allow us to have a single source of truth for all connector-related information. + +This transition is currently in progress. Once it is complete, the `icon` field in the `metadata.yaml` file will be removed, and the `icon.svg` file will be used instead. From 003ef5fc1e79a8ac59b380caa969d9212e398bf2 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Tue, 13 Jun 2023 12:08:12 +0300 Subject: [PATCH 07/18] Source: Iterable - read timeout added (#26459) * read timeout added Signed-off-by: Sergey Chvalyuk * iterable.md updated Signed-off-by: Sergey Chvalyuk --------- Signed-off-by: Sergey Chvalyuk --- .../connectors/source-iterable/Dockerfile | 2 +- .../connectors/source-iterable/metadata.yaml | 2 +- .../source-iterable/source_iterable/streams.py | 17 ++++++++++++++++- .../source-iterable/unit_tests/test_streams.py | 17 +++++++++++++++++ docs/integrations/sources/iterable.md | 5 +++-- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index ed6f0f878f590..de1c6a4560bd6 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.28 +LABEL io.airbyte.version=0.1.29 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/metadata.yaml b/airbyte-integrations/connectors/source-iterable/metadata.yaml index 79d9870e77be2..85cd7516a3a26 100644 --- a/airbyte-integrations/connectors/source-iterable/metadata.yaml +++ b/airbyte-integrations/connectors/source-iterable/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 - dockerImageTag: 0.1.28 + dockerImageTag: 0.1.29 dockerRepository: airbyte/source-iterable githubIssueLabel: source-iterable icon: iterable.svg diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index 8cc9bc44506d2..177c7987dd38d 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -88,6 +88,18 @@ def check_generic_error(self, response: requests.Response) -> bool: if response_json.get("code") in codes and msg_pattern in response_json.get("msg", ""): return True + def request_kwargs( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Mapping[str, Any]: + """ + https://requests.readthedocs.io/en/latest/user/advanced/#timeouts + https://github.com/airbytehq/oncall/issues/1985#issuecomment-1559276465 + """ + return {"timeout": (60, 300)} + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() or {} records = response_json.get(self.data_field, []) @@ -216,7 +228,10 @@ def request_kwargs( Passing stream=True argument to requests.session.send method to avoid loading whole analytics report content into memory. """ - return {"stream": True} + return { + **super().request_kwargs(stream_state, stream_slice, next_page_token), + "stream": True, + } def get_start_date(self, stream_state: Mapping[str, Any]) -> DateTime: stream_state = stream_state or {} diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index e38365bcc4125..cbf7c611059b7 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -3,6 +3,7 @@ # import json +from unittest.mock import MagicMock import pendulum import pytest @@ -249,3 +250,19 @@ def get_body(emails): records = list(read_full_refresh(stream)) assert [r["email"] for r in records] == ['user1', 'user2', 'user3', 'user5'] assert m.call_count == 3 + + +def test_retry_read_timeout(): + stream = Lists(authenticator=None) + stream._session.send = MagicMock(side_effect=requests.exceptions.ReadTimeout) + with pytest.raises(requests.exceptions.ReadTimeout): + list(read_full_refresh(stream)) + stream._session.send.call_args[1] == {'timeout': (60, 300)} + assert stream._session.send.call_count == stream.max_retries + 1 + + stream = Campaigns(authenticator=None) + stream._session.send = MagicMock(side_effect=requests.exceptions.ConnectionError) + with pytest.raises(requests.exceptions.ConnectionError): + list(read_full_refresh(stream)) + stream._session.send.call_args[1] == {'timeout': (60, 300)} + assert stream._session.send.call_count == stream.max_retries + 1 diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index 3900607d44415..61deaa2a13b63 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -80,12 +80,13 @@ The Iterable source connector supports the following [sync modes](https://docs.a | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.29 | 2023-05-24 | [26459](https://github.com/airbytehq/airbyte/pull/26459) | Added requests reading timeout 300 seconds | | 0.1.28 | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream | | 0.1.27 | 2023-04-06 | [24962](https://github.com/airbytehq/airbyte/pull/24962) | `UserList` stream when meet `500 - Generic Error` will skip a broken slice and keep going with the next one | | 0.1.26 | 2023-03-10 | [23938](https://github.com/airbytehq/airbyte/pull/23938) | Improve retry for `500 - Generic Error` | | 0.1.25 | 2023-03-07 | [23821](https://github.com/airbytehq/airbyte/pull/23821) | Added retry for `500 - Generic Error`, increased max attempts number to `6` to handle `ChunkedEncodingError` | -| 0.1.24 | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification | -| 0.1.23 | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.24 | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification | +| 0.1.23 | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.22 | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak | | 0.1.21 | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery | | 0.1.20 | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors | From 6581b6992a36287fb6ec86151285fbff05a4ff24 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 13 Jun 2023 13:48:36 +0300 Subject: [PATCH 08/18] :bug: Source Snapchat Marketing: remove deprecated auth specification (#26358) * Connector health: source hubspot, gitlab, snapchat-marketing: fix builds * #26246 #26247 source Square, source Snapchat-Marketing: remove deprecated authSpecification in favour of advancedAuth * #26246, #26247: upd changelogs * Automated Change * Automated Change * revert square changes * Automated Change * Automated Commit - Formatting Changes * Automated Commit - Format and Process Resources Changes --------- Co-authored-by: davydov-d --- ...onnectorRegistryDestinationDefinition.yaml | 2 +- .../ConnectorRegistrySourceDefinition.yaml | 2 +- .../source-snapchat-marketing/Dockerfile | 2 +- .../source-snapchat-marketing/metadata.yaml | 2 +- .../source_snapchat_marketing/spec.json | 42 ++++++++++++++++--- .../sources/snapchat-marketing.md | 39 ++++++++--------- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml index 35f266a56c415..3e99bf4ab83db 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml @@ -67,4 +67,4 @@ properties: allowedHosts: "$ref": AllowedHosts.yaml releases: - "$ref": ConnectorReleases.yaml + "$ref": ConnectorReleases.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml index 9b1640af0bb4a..49886b78e1c29 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml @@ -69,4 +69,4 @@ properties: description: Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach type: integer releases: - "$ref": ConnectorReleases.yaml + "$ref": ConnectorReleases.yaml diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile index c55aae2de4946..64dc6bd5a121f 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile @@ -25,5 +25,5 @@ COPY source_snapchat_marketing ./source_snapchat_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml index 9ddd952b3c5f5..4e63d0ba6b56f 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-snapchat-marketing githubIssueLabel: source-snapchat-marketing icon: snapchat.svg diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json index f1b5c99d70a9f..6d26cdd036b71 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json @@ -48,12 +48,42 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"] + } + } + } } } } diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index f1eab3a6fa3ce..73c07998cbe1a 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -111,22 +111,23 @@ Snapchat Marketing API has limitations to 1000 items per page. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------| -| 0.2.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema | -| 0.1.16 | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema | -| 0.1.15 | 2023-03-02 | [22869](https://github.com/airbytehq/airbyte/pull/22869) | Specified date formatting in specification | -| 0.1.14 | 2023-02-10 | [22808](https://github.com/airbytehq/airbyte/pull/22808) | Enable default `AvailabilityStrategy` | -| 0.1.13 | 2023-01-27 | [22023](https://github.com/airbytehq/airbyte/pull/22023) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.12 | 2023-01-11 | [21267](https://github.com/airbytehq/airbyte/pull/21267) | Fix parse empty error response | -| 0.1.11 | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error | -| 0.1.10 | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0 | -| 0.1.9 | 2022-12-14 | [20498](https://github.com/airbytehq/airbyte/pull/20498) | Fix output state when no records are read | -| 0.1.8 | 2022-10-05 | [17596](https://github.com/airbytehq/airbyte/pull/17596) | Retry 429 and 5xx errors when refreshing access token | -| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from specs | -| 0.1.5 | 2022-07-13 | [14577](https://github.com/airbytehq/airbyte/pull/14577) | Added stats streams hourly, daily, lifetime | -| 0.1.4 | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.3 | 2021-11-10 | [7811](https://github.com/airbytehq/airbyte/pull/7811) | Add oauth2.0, fix stream_state | -| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.1 | 2021-07-29 | [5072](https://github.com/airbytehq/airbyte/pull/5072) | Fix bug with incorrect stream\_state value | -| 0.1.0 | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843) | Initial release supporting the Snapchat Marketing API | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------| +| 0.3.0 | 2023-05-22 | [26358](https://github.com/airbytehq/airbyte/pull/26358) | Remove deprecated authSpecification in favour of advancedAuth | +| 0.2.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema | +| 0.1.16 | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema | +| 0.1.15 | 2023-03-02 | [22869](https://github.com/airbytehq/airbyte/pull/22869) | Specified date formatting in specification | +| 0.1.14 | 2023-02-10 | [22808](https://github.com/airbytehq/airbyte/pull/22808) | Enable default `AvailabilityStrategy` | +| 0.1.13 | 2023-01-27 | [22023](https://github.com/airbytehq/airbyte/pull/22023) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.12 | 2023-01-11 | [21267](https://github.com/airbytehq/airbyte/pull/21267) | Fix parse empty error response | +| 0.1.11 | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error | +| 0.1.10 | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0 | +| 0.1.9 | 2022-12-14 | [20498](https://github.com/airbytehq/airbyte/pull/20498) | Fix output state when no records are read | +| 0.1.8 | 2022-10-05 | [17596](https://github.com/airbytehq/airbyte/pull/17596) | Retry 429 and 5xx errors when refreshing access token | +| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from specs | +| 0.1.5 | 2022-07-13 | [14577](https://github.com/airbytehq/airbyte/pull/14577) | Added stats streams hourly, daily, lifetime | +| 0.1.4 | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.3 | 2021-11-10 | [7811](https://github.com/airbytehq/airbyte/pull/7811) | Add oauth2.0, fix stream_state | +| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.1 | 2021-07-29 | [5072](https://github.com/airbytehq/airbyte/pull/5072) | Fix bug with incorrect stream\_state value | +| 0.1.0 | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843) | Initial release supporting the Snapchat Marketing API | \ No newline at end of file From 9a040af21215864d3c75150fcedc5a109fc14b9b Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 13 Jun 2023 13:14:38 +0200 Subject: [PATCH 09/18] connectors-ci: run `dockerd` in a background task (#27267) * connectors-ci: long running but ephemeral dockerd service * Automated Commit - Formatting Changes * to revert * Revert "to revert" This reverts commit 168ed30695ec858c68055dba06bfd34e7a95b88d. --------- Co-authored-by: alafanechere --- .../pipelines/actions/environments.py | 21 ++------------ .../pipelines/builds/java_connectors.py | 1 - .../ci_connector_ops/pipelines/consts.py | 6 ++-- .../ci_connector_ops/pipelines/gradle.py | 9 +----- .../pipelines/pipelines/connectors.py | 29 ++++++++++++------- .../pipelines/tests/java_connectors.py | 10 ++----- 6 files changed, 28 insertions(+), 48 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py index 412860c8f263a..3c3fe157ea0f5 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py @@ -292,11 +292,6 @@ def with_global_dockerd_service(dagger_client: Client) -> Container: return ( dagger_client.container() .from_(consts.DOCKER_DIND_IMAGE) - .with_mounted_cache( - "/var/lib/docker", - dagger_client.cache_volume("docker-lib"), - sharing=CacheSharingMode.SHARED, - ) .with_mounted_cache( "/tmp", dagger_client.cache_volume("shared-tmp"), @@ -315,30 +310,23 @@ def with_bound_docker_host( Args: context (ConnectorContext): The current connector context. container (Container): The container to bind to the docker host. - shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None. - docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None. - Returns: Container: The container bound to the docker host. """ dockerd = context.dockerd_service docker_hostname = "global-docker-host" - bound = ( + return ( container.with_env_variable("DOCKER_HOST", f"tcp://{docker_hostname}:2375") .with_service_binding(docker_hostname, dockerd) .with_mounted_cache("/tmp", context.dagger_client.cache_volume("shared-tmp")) ) - return bound - def with_docker_cli(context: ConnectorContext) -> Container: """Create a container with the docker CLI installed and bound to a persistent docker host. Args: context (ConnectorContext): The current connector context. - shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None. - docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None. Returns: Container: A docker cli container bound to a docker host. @@ -357,7 +345,7 @@ async def with_connector_acceptance_test(context: ConnectorContext, connector_un Container: A container with connector acceptance tests installed. """ connector_under_test_image_name = context.connector.acceptance_test_config["connector_image"] - await load_image_to_docker_host(context, connector_under_test_image_tar, connector_under_test_image_name, docker_service_name="cat") + await load_image_to_docker_host(context, connector_under_test_image_tar, connector_under_test_image_name) if context.connector_acceptance_test_image.endswith(":dev"): cat_container = context.connector_acceptance_test_source_dir.docker_build() @@ -381,7 +369,6 @@ def with_gradle( context: ConnectorContext, sources_to_include: List[str] = None, bind_to_docker_host: bool = True, - docker_service_name: Optional[str] = "gradle", ) -> Container: """Create a container with Gradle installed and bound to a persistent docker host. @@ -389,7 +376,6 @@ def with_gradle( context (ConnectorContext): The current connector context. sources_to_include (List[str], optional): List of additional source path to mount to the container. Defaults to None. bind_to_docker_host (bool): Whether to bind the gradle container to a docker host. - docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to "gradle". Returns: Container: A container with Gradle installed and Java sources from the repository. @@ -443,14 +429,13 @@ def with_gradle( return openjdk_with_docker -async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str, docker_service_name: Optional[str] = None): +async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str): """Load a docker image tar archive to the docker host. Args: context (ConnectorContext): The current connector context. tar_file (File): The file object holding the docker image tar archive. image_tag (str): The tag to create on the image if it has no tag. - docker_service_name (str): Name of the docker service, useful for context isolation. """ # Hacky way to make sure the image is always loaded tar_name = f"{str(uuid.uuid4())}.tar" diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py index 032e113d0de5e..8f70354714f77 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py @@ -21,7 +21,6 @@ async def _run(self) -> StepResult: environments.with_gradle( self.context, self.build_include, - docker_service_name=self.docker_service_name, ) .with_mounted_directory(str(self.context.connector.code_directory), await self._get_patched_connector_dir()) .with_exec(self._get_gradle_command()) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py index c5150e05f45a6..47405322a2d1b 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py @@ -25,9 +25,9 @@ CI_CONNECTOR_OPS_SOURCE_PATH = "tools/ci_connector_ops" BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}") -DOCKER_VERSION = "20.10.23" -DOCKER_DIND_IMAGE = "docker:20-dind" -DOCKER_CLI_IMAGE = "docker:20-cli" +DOCKER_VERSION = "24.0.2" +DOCKER_DIND_IMAGE = "docker:24-dind" +DOCKER_CLI_IMAGE = "docker:24-cli" GRADLE_CACHE_PATH = "/root/.gradle/caches" GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1" GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache" diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py index 6a28a7180c49f..33c580f871252 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py @@ -10,7 +10,6 @@ from ci_connector_ops.pipelines import consts from ci_connector_ops.pipelines.actions import environments from ci_connector_ops.pipelines.bases import Step, StepResult -from ci_connector_ops.pipelines.utils import slugify from dagger import CacheVolume, Container, Directory @@ -33,10 +32,6 @@ class GradleTask(Step, ABC): "project(':airbyte-integrations:bases:base-normalization').airbyteDocker.output", ] - @property - def docker_service_name(self) -> str: - return slugify(f"gradle-{self.title}") - @property def connector_java_build_cache(self) -> CacheVolume: return self.context.dagger_client.cache_volume("connector_java_build_cache") @@ -81,9 +76,7 @@ def _get_gradle_command(self, extra_options: Tuple[str] = ("--no-daemon", "--sca async def _run(self) -> StepResult: connector_under_test = ( - environments.with_gradle( - self.context, self.build_include, docker_service_name=self.docker_service_name, bind_to_docker_host=self.BIND_TO_DOCKER_HOST - ) + environments.with_gradle(self.context, self.build_include, bind_to_docker_host=self.BIND_TO_DOCKER_HOST) .with_mounted_directory(str(self.context.connector.code_directory), await self._get_patched_connector_dir()) # Disable the Ryuk container because it needs privileged docker access that does not work: .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true") diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py index ab41159f1e487..ca5434cdbdd12 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py @@ -78,18 +78,25 @@ async def run_connectors_pipelines( default_connectors_semaphore = anyio.Semaphore(concurrency) async with dagger.Connection(Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client: + # HACK: This is to get a long running dockerd service to be shared across all the connectors pipelines + # Using the "normal" service binding leads to restart of dockerd during pipeline run that can cause corrupted docker state + # See https://github.com/airbytehq/airbyte/issues/27233 dockerd_service = environments.with_global_dockerd_service(dagger_client) - async with anyio.create_task_group() as tg: - for context in contexts: - context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") - context.dockerd_service = dockerd_service - tg.start_soon( - connector_pipeline, - context, - CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), - *args, - ) - + async with anyio.create_task_group() as tg_main: + tg_main.start_soon(dockerd_service.exit_code) + await anyio.sleep(10) # Wait for the docker service to be ready + async with anyio.create_task_group() as tg_connectors: + for context in contexts: + context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") + context.dockerd_service = dockerd_service + tg_connectors.start_soon( + connector_pipeline, + context, + CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), + *args, + ) + # When the connectors pipelines are done, we can stop the dockerd service + tg_main.cancel_scope.cancel() await run_report_complete_pipeline(dagger_client, contexts) return contexts diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py index 1d5aef1c275ea..671d708ccf4a1 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py @@ -7,7 +7,6 @@ from typing import List, Optional import anyio -from ci_connector_ops.pipelines.helpers.steps import run_steps from ci_connector_ops.pipelines.actions import environments, secrets from ci_connector_ops.pipelines.bases import StepResult, StepStatus from ci_connector_ops.pipelines.builds import LOCAL_BUILD_PLATFORM @@ -15,6 +14,7 @@ from ci_connector_ops.pipelines.builds.normalization import BuildOrPullNormalization from ci_connector_ops.pipelines.contexts import ConnectorContext from ci_connector_ops.pipelines.gradle import GradleTask +from ci_connector_ops.pipelines.helpers.steps import run_steps from ci_connector_ops.pipelines.tests.common import AcceptanceTests from ci_connector_ops.pipelines.utils import export_container_to_tarball from dagger import File, QueryError @@ -34,17 +34,13 @@ class IntegrationTestJava(GradleTask): async def _load_normalization_image(self, normalization_tar_file: File): normalization_image_tag = f"{self.context.connector.normalization_repository}:dev" self.context.logger.info("Load the normalization image to the docker host.") - await environments.load_image_to_docker_host( - self.context, normalization_tar_file, normalization_image_tag, docker_service_name=self.docker_service_name - ) + await environments.load_image_to_docker_host(self.context, normalization_tar_file, normalization_image_tag) self.context.logger.info("Successfully loaded the normalization image to the docker host.") async def _load_connector_image(self, connector_tar_file: File): connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev" self.context.logger.info("Load the connector image to the docker host") - await environments.load_image_to_docker_host( - self.context, connector_tar_file, connector_image_tag, docker_service_name=self.docker_service_name - ) + await environments.load_image_to_docker_host(self.context, connector_tar_file, connector_image_tag) self.context.logger.info("Successfully loaded the connector image to the docker host.") async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[File]) -> StepResult: From 25ef3a2686617fe0549871d7453b6da70a546db6 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 13 Jun 2023 13:17:05 +0200 Subject: [PATCH 10/18] connectors-ci: disable format check on python connectors (#27301) * disable format check on python connectors * Automated Commit - Formatting Changes * disable format check on python connectors --------- Co-authored-by: alafanechere --- .../ci_connector_ops/pipelines/tests/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py index a8b41bafa8873..9aaee423df6a6 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py @@ -22,8 +22,9 @@ ConnectorLanguage.JAVA: java_connectors.run_all_tests, }, "run_code_format_checks": { - ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, - ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, + # TODO: re-enable when we have a code formatter + # ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, + # ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks }, } From f48849fdb4f687f64774a291c96c83d9153fce6b Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Tue, 13 Jun 2023 08:40:55 -0400 Subject: [PATCH 11/18] [ISSUE #26909] adding message repository (#27158) * [ISSUE #26909] adding message repository * Automated Commit - Formatting Changes * [ISSUE #26909] improve entrypoint error handling * format CDK * [ISSUE #26909] adding an integration test --- .../python/airbyte_cdk/config_observation.py | 12 ++- .../connector_builder/message_grouper.py | 2 +- airbyte-cdk/python/airbyte_cdk/entrypoint.py | 53 ++++++++---- .../airbyte_cdk/sources/abstract_source.py | 13 +++ .../manifest_declarative_source.py | 6 ++ .../parsers/model_to_component_factory.py | 9 +- .../airbyte_cdk/sources/message/__init__.py | 7 ++ .../airbyte_cdk/sources/message/repository.py | 36 ++++++++ .../http/requests_native_auth/oauth.py | 11 ++- .../test_connector_builder_handler.py | 37 +++++++- .../connector_builder/test_message_grouper.py | 12 --- .../unit_tests/sources/message/__init__.py | 0 .../sources/message/test_repository.py | 65 ++++++++++++++ .../test_requests_native_auth.py | 28 +++++- .../sources/test_abstract_source.py | 48 ++++++++++- .../unit_tests/test_config_observation.py | 14 ++- .../python/unit_tests/test_entrypoint.py | 86 ++++++++++++++++--- 17 files changed, 383 insertions(+), 56 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/message/repository.py create mode 100644 airbyte-cdk/python/unit_tests/sources/message/__init__.py create mode 100644 airbyte-cdk/python/unit_tests/sources/message/test_repository.py diff --git a/airbyte-cdk/python/airbyte_cdk/config_observation.py b/airbyte-cdk/python/airbyte_cdk/config_observation.py index 3ead7b295945e..c7a9b89609626 100644 --- a/airbyte-cdk/python/airbyte_cdk/config_observation.py +++ b/airbyte-cdk/python/airbyte_cdk/config_observation.py @@ -68,10 +68,18 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str, def emit_configuration_as_airbyte_control_message(config: MutableMapping): + """ + WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism. + See the airbyte_cdk.sources.message package + """ + airbyte_message = create_connector_config_control_message(config) + print(airbyte_message.json(exclude_unset=True)) + + +def create_connector_config_control_message(config): control_message = AirbyteControlMessage( type=OrchestratorType.CONNECTOR_CONFIG, emitted_at=time.time() * 1000, connectorConfig=AirbyteControlConnectorConfigMessage(config=config), ) - airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message) - print(airbyte_message.json(exclude_unset=True)) + return AirbyteMessage(type=Type.CONTROL, control=control_message) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index 3a7d6b91725a6..96661b511ee9a 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -82,7 +82,7 @@ def get_message_groups( inferred_schema=schema_inferrer.get_stream_schema( configured_catalog.streams[0].stream.name ), # The connector builder currently only supports reading from a single stream at a time - latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config), + latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None, inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(), ) diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index 40604d4be95b6..4a16b534e23ca 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -77,27 +77,32 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: else: self.logger.setLevel(logging.INFO) - # todo: add try catch for exceptions with different exit codes source_spec: ConnectorSpecification = self.source.spec(self.logger) - with tempfile.TemporaryDirectory() as temp_dir: - if cmd == "spec": - message = AirbyteMessage(type=Type.SPEC, spec=source_spec) - yield message.json(exclude_unset=True) - else: - raw_config = self.source.read_config(parsed_args.config) - config = self.source.configure(raw_config, temp_dir) - - if cmd == "check": - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config)) - elif cmd == "discover": - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config)) - elif cmd == "read": - config_catalog = self.source.read_catalog(parsed_args.catalog) - state = self.source.read_state(parsed_args.state) - - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state)) + try: + with tempfile.TemporaryDirectory() as temp_dir: + if cmd == "spec": + message = AirbyteMessage(type=Type.SPEC, spec=source_spec) + yield from [ + self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source) + ] + yield self.airbyte_message_to_string(message) else: - raise Exception("Unexpected command " + cmd) + raw_config = self.source.read_config(parsed_args.config) + config = self.source.configure(raw_config, temp_dir) + + if cmd == "check": + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config)) + elif cmd == "discover": + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config)) + elif cmd == "read": + config_catalog = self.source.read_catalog(parsed_args.catalog) + state = self.source.read_state(parsed_args.state) + + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state)) + else: + raise Exception("Unexpected command " + cmd) + finally: + yield from [self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)] def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]: self.set_up_secret_filter(config, source_spec.connectionSpecification) @@ -106,6 +111,7 @@ def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterabl except AirbyteTracedException as traced_exc: connection_status = traced_exc.as_connection_status_message() if connection_status: + yield from self._emit_queued_messages(self.source) yield connection_status return @@ -115,6 +121,7 @@ def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterabl else: self.logger.error("Check failed") + yield from self._emit_queued_messages(self.source) yield AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result) def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]: @@ -122,6 +129,8 @@ def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iter if self.source.check_config_against_spec: self.validate_connection(source_spec, config) catalog = self.source.discover(self.logger, config) + + yield from self._emit_queued_messages(self.source) yield AirbyteMessage(type=Type.CATALOG, catalog=catalog) def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TCatalog, state: TState) -> Iterable[AirbyteMessage]: @@ -130,6 +139,7 @@ def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TC self.validate_connection(source_spec, config) yield from self.source.read(self.logger, config, catalog, state) + yield from self._emit_queued_messages(self.source) @staticmethod def validate_connection(source_spec: ConnectorSpecification, config: Mapping[str, Any]) -> None: @@ -149,6 +159,11 @@ def set_up_secret_filter(config, connection_specification: Mapping[str, Any]): def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str: return airbyte_message.json(exclude_unset=True) + def _emit_queued_messages(self, source) -> Iterable[AirbyteMessage]: + if hasattr(source, "message_repository") and source.message_repository: + yield from source.message_repository.consume_queue() + return + def launch(source: Source, args: List[str]): source_entrypoint = AirbyteEntrypoint(source) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index c6fc000b149d8..18daa6a5204b1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -22,6 +22,7 @@ ) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.source import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.core import StreamData @@ -130,6 +131,7 @@ def read( yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) raise e except Exception as e: + yield from self._emit_queued_messages() logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) @@ -198,6 +200,7 @@ def _read_stream( logger.info(f"Marking stream {stream_name} as RUNNING") # If we just read the first record of the stream, emit the transition to the RUNNING state yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING) + yield from self._emit_queued_messages() yield record logger.info(f"Read {record_counter} records from {stream_name} stream") @@ -264,6 +267,7 @@ def _read_incremental( record_counter = 0 for message_counter, record_data_or_message in enumerate(records, start=1): message = self._get_message(record_data_or_message, stream_instance) + yield from self._emit_queued_messages() yield message if message.type == MessageType.RECORD: record = message.record @@ -298,6 +302,11 @@ def should_log_slice_message(self, logger: logging.Logger): """ return logger.isEnabledFor(logging.DEBUG) + def _emit_queued_messages(self): + if self.message_repository: + yield from self.message_repository.consume_queue() + return + def _read_full_refresh( self, logger: logging.Logger, @@ -357,3 +366,7 @@ def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage] return record_data_or_message else: return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema()) + + @property + def message_repository(self) -> Union[None, MessageRepository]: + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 1dcaccbfa4f4f..ee014912c68ab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -26,6 +26,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory from airbyte_cdk.sources.declarative.types import ConnectionDefinition +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.core import Stream from jsonschema.exceptions import ValidationError from jsonschema.validators import validate @@ -61,6 +62,7 @@ def __init__( self._debug = debug self._emit_connector_builder_messages = emit_connector_builder_messages self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages) + self._message_repository = self._constructor.get_message_repository() self._validate_source() @@ -68,6 +70,10 @@ def __init__( def resolved_manifest(self) -> Mapping[str, Any]: return self._source_config + @property + def message_repository(self) -> Union[None, MessageRepository]: + return self._message_repository + @property def connection_checker(self) -> ConnectionChecker: check = self._source_config["check"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 487708a084844..a2018ca9f4ba2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -100,6 +100,7 @@ from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.message import InMemoryMessageRepository from pydantic import BaseModel ComponentDefinition: Union[Literal, Mapping, List] @@ -121,6 +122,7 @@ def __init__( self._limit_slices_fetched = limit_slices_fetched self._emit_connector_builder_messages = emit_connector_builder_messages self._disable_retries = disable_retries + self._message_repository = InMemoryMessageRepository() def _init_mappings(self): self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = { @@ -675,8 +677,7 @@ def create_no_auth(model: NoAuthModel, config: Config, **kwargs) -> NoAuth: def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> NoPagination: return NoPagination(parameters={}) - @staticmethod - def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator: + def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator: if model.refresh_token_updater: return DeclarativeSingleUseRefreshTokenOauth2Authenticator( config, @@ -693,6 +694,7 @@ def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, * refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters).eval(config), scopes=model.scopes, token_expiry_date_format=model.token_expiry_date_format, + message_repository=self._message_repository, ) return DeclarativeOauth2Authenticator( access_token_name=model.access_token_name, @@ -845,3 +847,6 @@ def create_wait_until_time_from_header( return WaitUntilTimeFromHeaderBackoffStrategy( header=model.header, parameters=model.parameters, config=config, min_wait=model.min_wait, regex=model.regex ) + + def get_message_repository(self): + return self._message_repository diff --git a/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py new file mode 100644 index 0000000000000..5dec9072fed32 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from .repository import InMemoryMessageRepository, MessageRepository + +__all__ = ["InMemoryMessageRepository", "MessageRepository"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py new file mode 100644 index 0000000000000..17ae427bf0781 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Iterable + +from airbyte_cdk.models import AirbyteMessage, Type + + +class MessageRepository(ABC): + @abstractmethod + def emit_message(self, message: AirbyteMessage) -> None: + raise NotImplementedError() + + @abstractmethod + def consume_queue(self) -> Iterable[AirbyteMessage]: + raise NotImplementedError() + + +class InMemoryMessageRepository(MessageRepository): + def __init__(self): + self._message_queue = [] + + def emit_message(self, message: AirbyteMessage) -> None: + """ + :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more + work and therefore this work has been postponed + """ + if message.type != Type.CONTROL: + raise ValueError("As of today, only AirbyteControlMessages are supported as part of the InMemoryMessageRepository") + self._message_queue.append(message) + + def consume_queue(self) -> Iterable[AirbyteMessage]: + while self._message_queue: + yield self._message_queue.pop(0) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 1a38eeb0f64f8..29e8a544b02d4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -6,7 +6,8 @@ import dpath import pendulum -from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message +from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator @@ -115,6 +116,7 @@ def __init__( refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), token_expiry_date_format: Optional[str] = None, + message_repository: MessageRepository = None, ): """ @@ -144,6 +146,7 @@ def __init__( self._token_expiry_date_format = token_expiry_date_format self._refresh_token_name = refresh_token_name self._connector_config = connector_config + self._message_repository = message_repository super().__init__( token_refresh_endpoint, self.get_client_id(), @@ -211,7 +214,11 @@ def get_access_token(self) -> str: self.access_token = new_access_token self.set_refresh_token(new_refresh_token) self.set_token_expiry_date(new_token_expiry_date) - emit_configuration_as_airbyte_control_message(self._connector_config) + if self._message_repository: + self._message_repository.emit_message(create_connector_config_control_message(self._connector_config)) + else: + # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message + emit_configuration_as_airbyte_control_message(self._connector_config) return self.access_token def refresh_access_token(self) -> Tuple[str, str, str]: diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index 2cd6da279fe73..376571e02a23f 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -176,9 +176,9 @@ def invalid_config_file(tmp_path): def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle: handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)]) - assert patch.call_count == 1 + assert patched_handle.call_count == 1 def test_handle_test_read(valid_read_config_file, configured_catalog): @@ -384,6 +384,37 @@ def test_read(): assert output_record == expected_airbyte_message +def test_config_update(): + manifest = copy.deepcopy(MANIFEST) + manifest["definitions"]["retriever"]["requester"]["authenticator"] = { + "type": "OAuthAuthenticator", + "token_refresh_endpoint": "https://oauth.endpoint.com/tokens/bearer", + "client_id": "{{ config['credentials']['client_id'] }}", + "client_secret": "{{ config['credentials']['client_secret'] }}", + "refresh_token": "{{ config['credentials']['refresh_token'] }}", + "refresh_token_updater": {} + } + config = copy.deepcopy(TEST_READ_CONFIG) + config["__injected_declarative_manifest"] = manifest + config["credentials"] = { + "client_id": "a client id", + "client_secret": "a client secret", + "refresh_token": "a refresh token", + } + source = ManifestDeclarativeSource(manifest) + + refresh_request_response = { + "access_token": "an updated access token", + "refresh_token": "an updated refresh token", + "expires_in": 3600, + } + with patch("airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", return_value=refresh_request_response): + output = handle_connector_builder_request( + source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), TestReadLimits() + ) + assert output.record.data["latest_config_update"] + + @patch("traceback.TracebackException.from_exception") def test_read_returns_error_response(mock_from_exception): class MockManifestDeclarativeSource: @@ -413,7 +444,7 @@ def check_config_against_spec(self): test_read_limit_reached=False, inferred_schema=None, inferred_datetime_formats={}, - latest_config_update={}) + latest_config_update=None) expected_message = AirbyteMessage( type=MessageType.RECORD, diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index 72bfe387b11ee..f5dda8fed77e0 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -554,18 +554,6 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin assert stream_read.latest_config_update == updated_config -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_given_no_control_message_then_use_in_memory_config_change_as_update(mock_entrypoint_read): - mock_source = make_mock_source(mock_entrypoint_read, iter(any_request_and_response_with_a_record())) - connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} - stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=full_config, configured_catalog=create_configured_catalog("hashiras") - ) - - assert stream_read.latest_config_update == CONFIG - - @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read): earliest = 0 diff --git a/airbyte-cdk/python/unit_tests/sources/message/__init__.py b/airbyte-cdk/python/unit_tests/sources/message/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py new file mode 100644 index 0000000000000..5c0f3a9d90b03 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteLogMessage, + AirbyteMessage, + Level, + OrchestratorType, + Type, +) +from airbyte_cdk.sources.message import InMemoryMessageRepository + +A_CONTROL = AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}), +) +ANOTHER_CONTROL = AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"another config": "another value"}), +) + + +def test_given_no_messages_when_consume_queue_then_return_empty(): + repo = InMemoryMessageRepository() + messages = list(repo.consume_queue()) + assert messages == [] + + +def test_given_messages_when_consume_queue_then_return_messages(): + repo = InMemoryMessageRepository() + first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL) + repo.emit_message(first_message) + second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL) + repo.emit_message(second_message) + + messages = repo.consume_queue() + + assert list(messages) == [first_message, second_message] + + +def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_queue(): + repo = InMemoryMessageRepository() + first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL) + repo.emit_message(first_message) + second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL) + repo.emit_message(second_message) + + message_generator = repo.consume_queue() + consumed_message = next(message_generator) + assert consumed_message == first_message + + second_message_generator = repo.consume_queue() + assert list(second_message_generator) == [second_message] + + +def test_given_message_is_not_control_message_when_emit_message_then_raise_error(): + repo = InMemoryMessageRepository() + with pytest.raises(ValueError): + repo.emit_message(AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="any log message"))) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index c01213805a2d0..dc93c41529c42 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -4,11 +4,13 @@ import json import logging +from unittest.mock import Mock import freezegun import pendulum import pytest import requests +from airbyte_cdk.models import OrchestratorType, Type from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, MultipleTokenAuthenticator, @@ -243,7 +245,7 @@ def test_init(self, connector_config): ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"), ] ) - def test_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config): + def test_given_no_message_repository_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", @@ -270,6 +272,30 @@ def test_get_access_token(self, test_name, expires_in_value, expiry_date_format, assert not captured.out assert authenticator.access_token == access_token == "new_access_token" + def test_given_message_repository_when_get_access_token_emit_message(self, mocker, connector_config): + message_repository = Mock() + authenticator = SingleUseRefreshTokenOauth2Authenticator( + connector_config, + token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], + token_expiry_date_format="YYYY-MM-DD", + message_repository=message_repository, + ) + authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", "2023-04-04", "new_refresh_token")) + authenticator.token_has_expired = mocker.Mock(return_value=True) + + authenticator.get_access_token() + + emitted_message = message_repository.emit_message.call_args_list[0].args[0] + assert emitted_message.type == Type.CONTROL + assert emitted_message.control.type == OrchestratorType.CONNECTOR_CONFIG + assert emitted_message.control.connectorConfig.config["credentials"]["access_token"] == "new_access_token" + assert emitted_message.control.connectorConfig.config["credentials"]["refresh_token"] == "new_refresh_token" + assert emitted_message.control.connectorConfig.config["credentials"]["token_expiry_date"] == "2023-04-04T00:00:00+00:00" + assert emitted_message.control.connectorConfig.config["credentials"]["client_id"] == "my_client_id" + assert emitted_message.control.connectorConfig.config["credentials"]["client_secret"] == "my_client_secret" + def test_refresh_access_token(self, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index aaa347ed351e3..7bbcc0e1813e4 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -7,7 +7,7 @@ import logging from collections import defaultdict from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from unittest.mock import call +from unittest.mock import Mock, call import pytest from airbyte_cdk.models import ( @@ -37,9 +37,11 @@ from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from pytest import fixture logger = logging.getLogger("airbyte") @@ -50,10 +52,12 @@ def __init__( check_lambda: Callable[[], Tuple[bool, Optional[Any]]] = None, streams: List[Stream] = None, per_stream: bool = True, + message_repository: MessageRepository = None ): self._streams = streams self.check_lambda = check_lambda self.per_stream = per_stream + self._message_repository = message_repository def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: if self.check_lambda: @@ -69,6 +73,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def per_stream_state_enabled(self) -> bool: return self.per_stream + @property + def message_repository(self): + return self._message_repository + class StreamNoStateMethod(Stream): name = "managers" @@ -97,6 +105,16 @@ def state(self, value: MutableMapping[str, Any]): self._cursor_value = value.get(self.cursor_field, self.start_date) +MESSAGE_FROM_REPOSITORY = Mock() + + +@fixture +def message_repository(): + message_repository = Mock(spec=MessageRepository) + message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]] + return message_repository + + def test_successful_check(): """Tests that if a source returns TRUE for the connection check the appropriate connectionStatus success message is returned""" expected = AirbyteConnectionStatus(status=Status.SUCCEEDED) @@ -221,6 +239,34 @@ def test_read_nonexistent_stream_raises_exception(mocker): list(src.read(logger, {}, catalog)) +def test_read_stream_emits_repository_message_before_record(mocker, message_repository): + stream = MockStream(name="my_stream") + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(MockStream, "read_records", side_effect=[[{"a record": "a value"}, {"another record": "another value"}]]) + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []] + + source = MockSource(streams=[stream], message_repository=message_repository) + + messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]))) + + assert messages.count(MESSAGE_FROM_REPOSITORY) == 1 + record_messages = (message for message in messages if message.type == Type.RECORD) + assert all(messages.index(MESSAGE_FROM_REPOSITORY) < messages.index(record) for record in record_messages) + + +def test_read_stream_emits_repository_message_on_error(mocker, message_repository): + stream = MockStream(name="my_stream") + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(MockStream, "read_records", side_effect=RuntimeError("error")) + message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]] + + source = MockSource(streams=[stream], message_repository=message_repository) + + with pytest.raises(RuntimeError): + messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]))) + assert MESSAGE_FROM_REPOSITORY in messages + + def test_read_stream_with_error_gets_display_message(mocker): stream = MockStream(name="my_stream") diff --git a/airbyte-cdk/python/unit_tests/test_config_observation.py b/airbyte-cdk/python/unit_tests/test_config_observation.py index 50791080cf185..a182854493dcf 100644 --- a/airbyte-cdk/python/unit_tests/test_config_observation.py +++ b/airbyte-cdk/python/unit_tests/test_config_observation.py @@ -6,7 +6,8 @@ import time import pytest -from airbyte_cdk.config_observation import ConfigObserver, ObservedDict, observe_connector_config +from airbyte_cdk.config_observation import ConfigObserver, ObservedDict, create_connector_config_control_message, observe_connector_config +from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, OrchestratorType, Type class TestObservedDict: @@ -74,3 +75,14 @@ def test_observe_already_observed_config(): observed_config = observe_connector_config({"foo": "bar"}) with pytest.raises(ValueError): observe_connector_config(observed_config) + + +def test_create_connector_config_control_message(): + A_CONFIG = {"config key": "config value"} + + message = create_connector_config_control_message(A_CONFIG) + + assert message.type == Type.CONTROL + assert message.control.type == OrchestratorType.CONNECTOR_CONFIG + assert message.control.connectorConfig == AirbyteControlConnectorConfigMessage(config=A_CONFIG) + assert message.control.emitted_at is not None diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index ab31a784d9467..047e94adfa2cc 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -14,10 +14,13 @@ from airbyte_cdk.models import ( AirbyteCatalog, AirbyteConnectionStatus, + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, AirbyteMessage, AirbyteRecordMessage, AirbyteStream, ConnectorSpecification, + OrchestratorType, Status, SyncMode, Type, @@ -35,6 +38,10 @@ def discover(self, **kwargs): def check(self, **kwargs): pass + @property + def message_repository(self): + pass + def _as_arglist(cmd: str, named_args: Mapping[str, Any]) -> List[str]: out = [cmd] @@ -53,8 +60,21 @@ def spec_mock(mocker): return mock +MESSAGE_FROM_REPOSITORY = AirbyteMessage( + type=Type.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=10, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"any config": "a config value"}), + ) +) + + @pytest.fixture -def entrypoint() -> AirbyteEntrypoint: +def entrypoint(mocker) -> AirbyteEntrypoint: + message_repository = MagicMock() + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []] + mocker.patch.object(MockSource, "message_repository", new_callable=mocker.PropertyMock, return_value=message_repository) return AirbyteEntrypoint(MockSource()) @@ -124,7 +144,10 @@ def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): parsed_args = Namespace(command="spec") expected = ConnectorSpecification(connectionSpecification={"hi": "hi"}) mocker.patch.object(MockSource, "spec", return_value=expected) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages @pytest.fixture @@ -158,41 +181,80 @@ def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, sch messages = list(entrypoint.run(parsed_args)) if config_valid: - assert [_wrap_message(check_value)] == messages + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages else: - assert len(messages) == 1 - airbyte_message = AirbyteMessage.parse_raw(messages[0]) - assert airbyte_message.type == Type.CONNECTION_STATUS - assert airbyte_message.connectionStatus.status == Status.FAILED - assert airbyte_message.connectionStatus.message.startswith("Config validation error:") + assert len(messages) == 2 + assert messages[0] == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) + connection_status_message = AirbyteMessage.parse_raw(messages[1]) + assert connection_status_message.type == Type.CONNECTION_STATUS + assert connection_status_message.connectionStatus.status == Status.FAILED + assert connection_status_message.connectionStatus.message.startswith("Config validation error:") def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="check", config="config_path") check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED) mocker.patch.object(MockSource, "check", return_value=check_value) - assert [_wrap_message(check_value)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages assert spec_mock.called +def test_run_check_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="check", config="config_path") + mocker.patch.object(MockSource, "check", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="discover", config="config_path") expected = AirbyteCatalog(streams=[AirbyteStream(name="stream", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh])]) mocker.patch.object(MockSource, "discover", return_value=expected) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages assert spec_mock.called +def test_run_discover_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="discover", config="config_path") + mocker.patch.object(MockSource, "discover", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") expected = AirbyteRecordMessage(stream="stream", data={"data": "stuff"}, emitted_at=1) mocker.patch.object(MockSource, "read_state", return_value={}) mocker.patch.object(MockSource, "read_catalog", return_value={}) mocker.patch.object(MockSource, "read", return_value=[AirbyteMessage(record=expected, type=Type.RECORD)]) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [_wrap_message(expected), MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages assert spec_mock.called -def test_invalid_command(entrypoint: AirbyteEntrypoint, mocker, config_mock): +def test_run_read_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") + mocker.patch.object(MockSource, "read_state", return_value={}) + mocker.patch.object(MockSource, "read_catalog", return_value={}) + mocker.patch.object(MockSource, "read", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + +def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): with pytest.raises(Exception): list(entrypoint.run(Namespace(command="invalid", config="conf"))) From cdbedca86ff3bde745925c657898a5551d24e4b4 Mon Sep 17 00:00:00 2001 From: maxi297 Date: Tue, 13 Jun 2023 12:46:50 +0000 Subject: [PATCH 12/18] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version=20o?= =?UTF-8?q?f=20Airbyte=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 87f4bb4fa5d81..b324cb350d600 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.40.3 +current_version = 0.40.4 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index f80e6bc0b61f1..6dbad8bd23e4a 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.40.4 +Emit messages using message repository + ## 0.40.3 Add utils for inferring datetime formats diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 6283e46a7b239..5608dc3e90ffc 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.40.3 +RUN pip install --prefix=/install airbyte-cdk==0.40.4 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.40.3 +LABEL io.airbyte.version=0.40.4 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 4622524ae234a..3beb2e945e80c 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -17,7 +17,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.40.3", + version="0.40.4", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From f9bf2a344923673f483c07a90f17034e5d13ed70 Mon Sep 17 00:00:00 2001 From: Baz Date: Tue, 13 Jun 2023 17:22:51 +0300 Subject: [PATCH 13/18] =?UTF-8?q?=F0=9F=90=9B=20Source=20Monday:=20add=20`?= =?UTF-8?q?error=5Fhandler`=20for=20`403`,=20`500`=20HTTPError=20codes=20(?= =?UTF-8?q?#27244)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-monday/Dockerfile | 2 +- .../source-monday/acceptance-test-config.yml | 9 ++- .../connectors/source-monday/metadata.yaml | 2 +- .../source-monday/source_monday/manifest.yaml | 36 +++++++--- .../source_monday/schemas/boards.json | 70 +++++++++++++------ .../source_monday/schemas/items.json | 42 +++++++---- .../source_monday/schemas/teams.json | 8 ++- .../source_monday/schemas/updates.json | 26 ++++--- docs/integrations/sources/monday.md | 1 + 9 files changed, 136 insertions(+), 60 deletions(-) diff --git a/airbyte-integrations/connectors/source-monday/Dockerfile b/airbyte-integrations/connectors/source-monday/Dockerfile index 2782b1f5f25b2..8e7eb1c27d9fe 100644 --- a/airbyte-integrations/connectors/source-monday/Dockerfile +++ b/airbyte-integrations/connectors/source-monday/Dockerfile @@ -34,5 +34,5 @@ COPY source_monday ./source_monday ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/source-monday diff --git a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml index f401faf6dfdfc..d781a9a9e0ce2 100644 --- a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml @@ -18,14 +18,17 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + # `boards`, `items`, `updates` streams schemas were modified + # when `type: array` is used, the `items` should be set, instead of `properties`. + # this change applies for all configs bellow. backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.5" - config_path: "secrets/config_api_token.json" backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.5" - config_path: "secrets/config_oauth.json" backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.5" basic_read: tests: - config_path: "secrets/config_api_token.json" diff --git a/airbyte-integrations/connectors/source-monday/metadata.yaml b/airbyte-integrations/connectors/source-monday/metadata.yaml index cb9d187df5b36..c629ce1f3d944 100644 --- a/airbyte-integrations/connectors/source-monday/metadata.yaml +++ b/airbyte-integrations/connectors/source-monday/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 80a54ea2-9959-4040-aac1-eee42423ec9b - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 dockerRepository: airbyte/source-monday githubIssueLabel: source-monday icon: monday.svg diff --git a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml index 12a85bc7f42db..1db971eba1e6d 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml +++ b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml @@ -18,16 +18,36 @@ definitions: http_method: "GET" authenticator: type: BearerAuthenticator - api_token: "{{ config['credentials']['api_token'] if config['credentials']['auth_type'] == 'api_token' else config['credentials']['access_token'] if config['credentials']['auth_type'] == 'oauth2.0' else config.get('api_token', '') }}" + api_token: "{{ config.get('credentials', {}).get('api_token') if config.get('credentials', {}).get('auth_type') == 'api_token' else config.get('credentials', {}).get('access_token') if config.get('credentials', {}).get('auth_type') == 'oauth2.0' else config.get('api_token', '') }}" limit: "{{ parameters['items_per_page'] }}" error_handler: - type: "DefaultErrorHandler" - response_filters: - - predicate: "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" - action: RETRY - backoff_strategies: - - type: ConstantBackoffStrategy - backoff_time_in_seconds: 60 + type: CompositeErrorHandler + error_handlers: + - type: "DefaultErrorHandler" + response_filters: + - predicate: "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" + action: RETRY + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 60 + - type: "DefaultErrorHandler" + description: " + Ignore the slice when there is no access to the requested entity - 'code: 403, message: None'. + https://github.com/airbytehq/alpha-beta-issues/issues/846 + " + response_filters: + - http_codes: [ 403 ] + action: IGNORE + - type: "DefaultErrorHandler" + description: " + Retry when `Internal Server Error occures - `code: 500, message: Internal server error`. + https://github.com/airbytehq/alpha-beta-issues/issues/245 + " + response_filters: + - http_codes: [ 500 ] + action: RETRY + backoff_strategies: + - type: ExponentialBackoffStrategy default_paginator: type: "DefaultPaginator" pagination_strategy: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json index 69f57548890b6..7eea49a9bd4b8 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json @@ -5,26 +5,34 @@ "board_kind": { "type": ["null", "string"] }, "columns": { "type": ["null", "array"], - "properties": { - "archived": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "string"] }, - "settings_str": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "type": { "type": ["null", "string"] }, - "width": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "archived": { "type": ["null", "boolean"] }, + "id": { "type": ["null", "string"] }, + "settings_str": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "width": { "type": ["null", "integer"] } + } } }, - "communication": { "type": ["null", "object"] }, + "communication": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] }, "groups": { "type": ["null", "array"], - "properties": { - "archived": { "type": ["null", "boolean"] }, - "color": { "type": ["null", "string"] }, - "deleted": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "string"] }, - "position": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "archived": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "deleted": { "type": ["null", "boolean"] }, + "id": { "type": ["null", "string"] }, + "position": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } } }, "id": { "type": ["null", "string"] }, @@ -40,14 +48,22 @@ "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "tags": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } }, "top_group": { @@ -59,14 +75,22 @@ "updated_at": { "type": ["null", "string"], "format": "date-time" }, "updates": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } }, "views": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } }, "workspace": { diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json index 66763c3e21c28..ccd2cc5ed005b 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json @@ -4,8 +4,12 @@ "properties": { "assets": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "board": { @@ -16,13 +20,17 @@ }, "column_values": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] }, - "value": { "type": ["null", "object"] }, - "additional_info": { "type": ["null", "object"] }, - "text": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "type": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] }, + "additional_info": { "type": ["null", "string"] }, + "text": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] } + } } }, "created_at": { "type": ["null", "string"], "format": "date-time" }, @@ -44,15 +52,23 @@ "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "updated_at": { "type": ["null", "string"], "format": "date-time" }, "updates": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json index c409d05b6bbdb..16cb865fcc92a 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json @@ -7,8 +7,12 @@ "picture_url": { "type": ["null", "string"] }, "users": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json index c35d559861036..2e52854b80e48 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json @@ -4,8 +4,12 @@ "properties": { "assets": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "body": { "type": ["null", "string"] }, @@ -15,13 +19,17 @@ "item_id": { "type": ["null", "string"] }, "replies": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] }, - "creator_id": { "type": ["null", "integer"] }, - "created_at": { "type": ["null", "string"], "format": "date-time" }, - "text_body": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"], "format": "date-time" }, - "body": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "creator_id": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "text_body": { "type": ["null", "string"] }, + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "body": { "type": ["null", "string"] } + } } }, "text_body": { "type": ["null", "string"] }, diff --git a/docs/integrations/sources/monday.md b/docs/integrations/sources/monday.md index a69967de5892c..c0b4fe52a2bcc 100644 --- a/docs/integrations/sources/monday.md +++ b/docs/integrations/sources/monday.md @@ -57,6 +57,7 @@ The Monday connector should not run into Monday API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------| +| 0.2.6 | 2023-06-12 | [27244](https://github.com/airbytehq/airbyte/pull/27244) | Added http error handling for `403` and `500` HTTP errors | | 0.2.5 | 2023-05-22 | [225881](https://github.com/airbytehq/airbyte/pull/25881) | Fix pagination for the items stream | | 0.2.4 | 2023-04-26 | [25277](https://github.com/airbytehq/airbyte/pull/25277) | Increase row limit to 100 | | 0.2.3 | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version | From cc5a2fc4e84df5c08fa9dc8116a1a63842b4c69e Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Tue, 13 Jun 2023 18:55:17 +0300 Subject: [PATCH 14/18] =?UTF-8?q?=F0=9F=90=9B=20Source=20Drift:=20migrate?= =?UTF-8?q?=20to=20advancedAuth=20(#27202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove authSpecification in favour of advancedAuth in the specification --- .../connectors/source-drift/Dockerfile | 2 +- .../connectors/source-drift/metadata.yaml | 2 +- .../source-drift/source_drift/spec.json | 63 +++++++++++++++++-- docs/integrations/sources/drift.md | 13 ++-- 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile b/airbyte-integrations/connectors/source-drift/Dockerfile index 67d7bff67428e..173b3501ab5b8 100644 --- a/airbyte-integrations/connectors/source-drift/Dockerfile +++ b/airbyte-integrations/connectors/source-drift/Dockerfile @@ -34,5 +34,5 @@ COPY source_drift ./source_drift ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/source-drift diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index 2265f095dc69d..b55312410291e 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 445831eb-78db-4b1f-8f1f-0d96ad8739e2 - dockerImageTag: 0.2.6 + dockerImageTag: 0.2.7 dockerRepository: airbyte/source-drift githubIssueLabel: source-drift icon: drift.svg diff --git a/airbyte-integrations/connectors/source-drift/source_drift/spec.json b/airbyte-integrations/connectors/source-drift/source_drift/spec.json index 29d76c79575bf..54253e694599b 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/spec.json +++ b/airbyte-integrations/connectors/source-drift/source_drift/spec.json @@ -75,12 +75,63 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] + "advanced_auth": { + "auth_flow_type" : "oauth2.0", + "predicate_key" : [ + "credentials", + "credentials" + ], + "predicate_value" : "oauth2.0", + "oauth_config_specification" : { + "complete_oauth_output_specification" : { + "type" : "object", + "properties" : { + "access_token" : { + "type" : "string", + "path_in_connector_config" : [ + "credentials", + "access_token" + ] + }, + "refresh_token" : { + "type" : "string", + "path_in_connector_config" : [ + "credentials", + "refresh_token" + ] + } + } + }, + "complete_oauth_server_input_specification" : { + "type" : "object", + "properties" : { + "client_id" : { + "type" : "string" + }, + "client_secret" : { + "type" : "string" + } + } + }, + "complete_oauth_server_output_specification" : { + "type" : "object", + "properties" : { + "client_id" : { + "type" : "string", + "path_in_connector_config" : [ + "credentials", + "client_id" + ] + }, + "client_secret" : { + "type" : "string", + "path_in_connector_config" : [ + "credentials", + "client_secret" + ] + } + } + } } } } diff --git a/docs/integrations/sources/drift.md b/docs/integrations/sources/drift.md index 4456ec21e9e43..9604e519bdb79 100644 --- a/docs/integrations/sources/drift.md +++ b/docs/integrations/sources/drift.md @@ -49,9 +49,10 @@ The Drift connector should not run into Drift API limitations under normal usage ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.2.6 | 2023-03-07 | [23810](https://github.com/airbytehq/airbyte/pull/23810) | Prepare for cloud | -| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Updated titles and descriptions | -| 0.2.3 | 2021-10-25 | [7337](https://github.com/airbytehq/airbyte/pull/7337) | Added support of `OAuth 2.0` authorisation option | -| `0.2.3` | 2021-10-27 | [7247](https://github.com/airbytehq/airbyte/pull/7247) | Migrate to the CDK | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:--------------------------------------------------------------------| +| 0.2.7 | 2023-06-09 | [27202](https://github.com/airbytehq/airbyte/pull/27202) | Remove authSpecification in favour of advancedAuth in specification | +| 0.2.6 | 2023-03-07 | [23810](https://github.com/airbytehq/airbyte/pull/23810) | Prepare for cloud | +| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Updated titles and descriptions | +| 0.2.3 | 2021-10-25 | [7337](https://github.com/airbytehq/airbyte/pull/7337) | Added support of `OAuth 2.0` authorisation option | +| `0.2.3` | 2021-10-27 | [7247](https://github.com/airbytehq/airbyte/pull/7247) | Migrate to the CDK | From 01a8f19d83b041ba9d41c95e1b5bad182e74018a Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Tue, 13 Jun 2023 12:10:22 -0400 Subject: [PATCH 15/18] [ISSUE #26607] CATs: support list in state (#27306) --- .../connector_acceptance_test/config.py | 4 ++-- .../connector_acceptance_test/tests/test_incremental.py | 2 +- .../connector_acceptance_test/utils/json_schema_helper.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index b4ce92f9e6070..b4656464c7ff4 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -7,7 +7,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Generic, List, Mapping, Optional, Set, TypeVar +from typing import Generic, List, Mapping, Optional, Set, TypeVar, Union from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel @@ -160,7 +160,7 @@ class FutureStateConfig(BaseConfig): class IncrementalConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path - cursor_paths: Optional[Mapping[str, List[str]]] = Field( + cursor_paths: Optional[Mapping[str, List[Union[int, str]]]] = Field( description="For each stream, the path of its cursor field in the output state messages." ) future_state: Optional[FutureStateConfig] = Field(description="Configuration for the future state.") diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py index 9379e5db8e8b6..121d5159ffd0c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py @@ -155,7 +155,7 @@ def test_two_sequential_reads( inputs: IncrementalConfig, connector_config: SecretDict, configured_catalog_for_incremental: ConfiguredAirbyteCatalog, - cursor_paths: dict[str, list[str]], + cursor_paths: dict[str, list[Union[int, str]]], docker_runner: ConnectorRunner, ): threshold_days = getattr(inputs, "threshold_days") or 0 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py index fe959d79bdb40..170e65781d2c0 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py @@ -43,7 +43,7 @@ def _parse_value(self, value: Any) -> Any: return pendulum.parse(value) return value - def parse(self, record: Mapping[str, Any], path: Optional[List[str]] = None) -> Any: + def parse(self, record: Mapping[str, Any], path: Optional[List[Union[int, str]]] = None) -> Any: """Extract field value from the record and cast it to native type""" path = path or self.path value = reduce(lambda data, key: data[key], path, record) From baa1ce4bd169fdda748882fe06fb276df19de767 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Tue, 13 Jun 2023 18:11:53 +0200 Subject: [PATCH 16/18] Source Typeform, Source Mixpanel, Source Harvest, Source Google Ads, Source Confluence, Source Zendesk Support and Source Amplitude: fix builds (#27253) * Fix CAT for source typeform * Fix unittest for google ads * Fix CAT for mixpanel and harvest * Fix CAT for confluence and zendesk support * Fix CAT for amplitude --- .../connectors/source-amplitude/README.md | 4 +++- .../acceptance-test-config.yml | 1 + .../integration_tests/expected_records.jsonl | 2 +- .../unit_tests/test_streams.py | 1 + .../integration_tests/expected_records.jsonl | 14 ++++++------- .../source_harvest/schemas/invoices.json | 3 +++ .../acceptance-test-config.yml | 5 +++++ .../integration_tests/expected_records.jsonl | 21 +++++-------------- .../connectors/source-twitter/README.md | 4 +++- .../integration_tests/expected_records.jsonl | 5 +---- .../integration_tests/expected_records.jsonl | 4 ++-- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/source-amplitude/README.md b/airbyte-integrations/connectors/source-amplitude/README.md index 2318166113d0d..17b153e7bbe24 100644 --- a/airbyte-integrations/connectors/source-amplitude/README.md +++ b/airbyte-integrations/connectors/source-amplitude/README.md @@ -50,7 +50,9 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` To run your integration tests with Docker, run: ``` ./acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml index 3aac2082e2b90..f2d615d750d41 100644 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -36,6 +36,7 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 3600 full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl index a19d6fae9346e..4f3f1b6337d86 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl @@ -1,5 +1,5 @@ {"stream": "audit", "data": {"author": {"type": "user", "displayName": "System", "operations": null, "isExternalCollaborator": false, "accountType": "", "publicName": "Unknown user", "externalCollaborator": false}, "remoteAddress": "", "creationDate": "1682500900997", "summary": "User added to group", "description": "", "category": "Users and groups", "sysAdmin": false, "superAdmin": false, "affectedObject": {"name": "jira-servicemanagement-users-airbyteio:aab99a7c-3ce3-4123-b580-e4e00460754d", "objectType": "Group"}, "changedValues": [], "associatedObjects": [{"name": "Atlassian Assist", "objectType": "User"}]}, "emitted_at": 1683113208533} -{"stream": "pages", "data": {"id": "98487", "type": "page", "status": "current", "title": "Overview", "history": {"lastUpdated": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:30.734Z", "contributors": {"_expandable": {"publishers": ""}}, "_expandable": {"lastOwnedBy": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/history"}}, "version": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "descendants": {"comment": {"results": [], "start": 0, "limit": 25, "size": 0, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant/comment"}}, "_expandable": {"attachment": "/rest/api/content/98487/descendant/attachment", "page": "/rest/api/content/98487/descendant/page"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant"}}, "macroRenderedOutput": {}, "body": {"storage": {"value": "\n\n

Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

\n \n\n

\n \n\n

\ud83d\udcc4 Recent pages that I've worked on

\n \n\n 5\n \n\n 5titles\n \n\n

\ud83d\udd90 Get in touch

\n \n\n

\u2709\ufe0f Insert your email here

\n

\ud83d\udcbc Insert your LinkedIn URL here

\n \n\n

\ud83d\udd17 Insert your Twitter handle here

\n

\ud83d\udc64 Insert your Medium profile here

\n \n\n

End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

\n ", "representation": "storage", "embeddedContent": [], "_expandable": {"content": "/rest/api/content/98487"}}, "view": {"value": "
\n
\n
\n
\n\n\n

Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

\n
\n
\n
\n
\n\n\n

\n

\n
\n
\n
\n
\n
\n\n\n

\ud83d\udcc4 Recent pages that I've worked on

\n
\n
\n
\n
\n
\n
\n\n\n \n\n
\n\n

Recently Updated

\n
\n \n
\n
\n \n
\n
\n\n
\n
\n
\n
\n\n\n \n \n\n
\n

Blog Posts

\n \n
\n\n
\n
\n
\n
\n
\n
\n\n\n

\ud83d\udd90 Get in touch

\n
\n
\n
\n
\n
\n
\n\n\n

\u2709\ufe0f

\n

\ud83d\udcbc

\n
\n
\n
\n
\n\n\n

\ud83d\udd17

\n

\ud83d\udc64

\n
\n
\n
\n
\n
\n
\n\n\n

End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

\n
\n
\n
\n
", "representation": "view", "_expandable": {"webresource": "", "embeddedContent": "", "mediaToken": "", "content": "/rest/api/content/98487"}}, "_expandable": {"editor": "", "atlas_doc_format": "", "export_view": "", "styled_view": "", "dynamic": "", "editor2": "", "anonymous_export_view": ""}}, "extensions": {"position": 959}, "restrictions": {"read": {"operation": "read", "restrictions": {"user": {"results": [], "start": 0, "limit": 200, "size": 0}, "_expandable": {"group": ""}}, "_expandable": {"content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation/read"}}, "_expandable": {"update": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation"}}, "_expandable": {"childTypes": "", "container": "/rest/api/space/~5fc9e78d2730d800760becc4", "schedulePublishInfo": "", "metadata": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/98487/child", "ancestors": "", "space": "/rest/api/space/~5fc9e78d2730d800760becc4"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487", "tinyui": "/x/t4AB", "editui": "/pages/resumedraft.action?draftId=98487", "webui": "/spaces/~5fc9e78d2730d800760becc4/overview"}}, "emitted_at": 1686174842743} +{"stream": "pages", "data": {"id": "98487", "type": "page", "status": "current", "title": "Overview", "history": {"lastUpdated": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:30.734Z", "contributors": {"_expandable": {"publishers": ""}}, "_expandable": {"lastOwnedBy": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/history"}}, "version": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "descendants": {"comment": {"results": [], "start": 0, "limit": 25, "size": 0, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant/comment"}}, "_expandable": {"attachment": "/rest/api/content/98487/descendant/attachment", "page": "/rest/api/content/98487/descendant/page"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant"}}, "macroRenderedOutput": {}, "body": {"storage": {"value": "\n\n

Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

\n \n\n

\n \n\n

\ud83d\udcc4 Recent pages that I've worked on

\n \n\n 5\n \n\n 5titles\n \n\n

\ud83d\udd90 Get in touch

\n \n\n

\u2709\ufe0f Insert your email here

\n

\ud83d\udcbc Insert your LinkedIn URL here

\n \n\n

\ud83d\udd17 Insert your Twitter handle here

\n

\ud83d\udc64 Insert your Medium profile here

\n \n\n

End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

\n ", "representation": "storage", "embeddedContent": [], "_expandable": {"content": "/rest/api/content/98487"}}, "view": {"value": "
\n
\n
\n
\n\n\n

Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

\n
\n
\n
\n
\n\n\n

\n

\n
\n
\n
\n
\n
\n\n\n

\ud83d\udcc4 Recent pages that I've worked on

\n
\n
\n
\n
\n
\n
\n\n\n \n\n
\n\n

Recently Updated

\n
\n \n
\n
\n \n
\n
\n\n
\n
\n
\n
\n\n\n \n \n\n
\n

Blog Posts

\n \n
\n\n
\n
\n
\n
\n
\n
\n\n\n

\ud83d\udd90 Get in touch

\n
\n
\n
\n
\n
\n
\n\n\n

\u2709\ufe0f

\n

\ud83d\udcbc

\n
\n
\n
\n
\n\n\n

\ud83d\udd17

\n

\ud83d\udc64

\n
\n
\n
\n
\n
\n
\n\n\n

End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

\n
\n
\n
\n
", "representation": "view", "_expandable": {"webresource": "", "embeddedContent": "", "mediaToken": "", "content": "/rest/api/content/98487"}}, "_expandable": {"editor": "", "atlas_doc_format": "", "export_view": "", "styled_view": "", "dynamic": "", "editor2": "", "anonymous_export_view": ""}}, "extensions": {"position": 959}, "restrictions": {"read": {"operation": "read", "restrictions": {"user": {"results": [], "start": 0, "limit": 200, "size": 0}, "_expandable": {"group": ""}}, "_expandable": {"content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation/read"}}, "_expandable": {"update": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation"}}, "_expandable": {"childTypes": "", "container": "/rest/api/space/~5fc9e78d2730d800760becc4", "schedulePublishInfo": "", "metadata": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/98487/child", "ancestors": "", "space": "/rest/api/space/~5fc9e78d2730d800760becc4"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487", "tinyui": "/x/t4AB", "editui": "/pages/resumedraft.action?draftId=98487", "webui": "/spaces/~5fc9e78d2730d800760becc4/overview"}}, "emitted_at": 1686651430427} {"stream": "blog_posts", "data": {"id": "1343496", "type": "blogpost", "status": "current", "title": "Blog post integration tests", "history": {"lastUpdated": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-03-06T15:54:15.290Z", "friendlyWhen": "Mar 06, 2023", "message": "", "number": 1, "minorEdit": false, "syncRev": "0.confluence$content$1343496.18", "syncRevSource": "synchrony-ack", "confRev": "confluence$content$1343496.19", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/version/1"}}, "latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-03-06T15:54:14.990Z", "contributors": {"_expandable": {"publishers": ""}}, "_expandable": {"lastOwnedBy": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/history"}}, "version": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-03-06T15:54:15.290Z", "friendlyWhen": "Mar 06, 2023", "message": "", "number": 1, "minorEdit": false, "syncRev": "0.confluence$content$1343496.18", "syncRevSource": "synchrony-ack", "confRev": "confluence$content$1343496.19", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/version/1"}}, "descendants": {"comment": {"results": [], "start": 0, "limit": 25, "size": 0, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/descendant/comment"}}, "_expandable": {"attachment": "/rest/api/content/1343496/descendant/attachment"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/descendant"}}, "macroRenderedOutput": {}, "body": {"storage": {"value": "

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

", "representation": "storage", "embeddedContent": [], "_expandable": {"content": "/rest/api/content/1343496"}}, "view": {"value": "

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

", "representation": "view", "_expandable": {"webresource": "", "embeddedContent": "", "mediaToken": "", "content": "/rest/api/content/1343496"}}, "_expandable": {"editor": "", "atlas_doc_format": "", "export_view": "", "styled_view": "", "dynamic": "", "editor2": "", "anonymous_export_view": ""}}, "restrictions": {"read": {"operation": "read", "restrictions": {"user": {"results": [], "start": 0, "limit": 200, "size": 0}, "_expandable": {"group": ""}}, "_expandable": {"content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/restriction/byOperation/read"}}, "_expandable": {"update": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/restriction/byOperation"}}, "_expandable": {"childTypes": "", "container": "/rest/api/space/~5fc9e78d2730d800760becc4", "schedulePublishInfo": "", "metadata": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/1343496/child", "ancestors": "", "space": "/rest/api/space/~5fc9e78d2730d800760becc4"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496", "tinyui": "/x/CIAU", "editui": "/pages/resumedraft.action?draftId=1343496", "webui": "/spaces/~5fc9e78d2730d800760becc4/blog/2023/03/06/1343496/Blog+post+integration+tests"}}, "emitted_at": 1683113210956} {"stream": "space", "data": {"id": 196612, "key": "AIRBYTE", "name": "Airbyte", "icon": {"path": "/download/attachments/196611/AIRBYTE-default?version=1&modificationDate=1676624930960&cacheVersion=1&api=v2", "width": 48, "height": 48, "isDefault": false}, "description": {"plain": {"value": "", "representation": "plain", "embeddedContent": []}, "view": {"value": "", "representation": "view", "embeddedContent": []}}, "type": "global", "permissions": [{"id": 196617, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196626, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196631, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196661, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196663, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196678, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196686, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196732, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196735, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229395, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229519, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196630, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196645, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196666, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196675, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196676, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196680, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196697, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196733, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196741, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229389, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229513, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196619, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196621, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196691, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196693, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196695, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196714, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196719, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196729, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196744, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229401, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229525, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196616, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196627, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196651, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196677, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196683, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196710, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196722, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196723, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196746, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229383, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229507, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196644, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196668, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196679, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196699, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196700, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196711, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196720, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196727, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196747, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196628, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196632, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196639, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196649, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196673, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196698, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196702, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196709, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196739, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229413, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229537, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196633, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196652, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196667, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196672, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196687, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196705, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196713, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196721, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196740, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229425, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229549, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196620, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196624, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196641, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196653, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196660, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196685, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196718, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196726, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196734, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229419, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229543, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196640, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196670, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196701, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196704, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196716, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196731, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196736, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196743, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196745, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196637, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196648, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196669, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196671, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196689, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196690, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196712, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196717, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196737, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229407, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229531, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196615, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196625, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196638, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196642, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196646, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196650, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196664, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196688, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196707, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196623, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196634, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196635, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196658, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196662, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196692, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196708, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196725, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229431, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229555, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196647, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196657, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196659, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196681, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196684, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196706, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196724, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196728, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196742, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229377, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229501, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}], "status": "current", "_expandable": {"settings": "/rest/api/space/AIRBYTE/settings", "metadata": "", "operations": "", "lookAndFeel": "/rest/api/settings/lookandfeel?spaceKey=AIRBYTE", "identifiers": "", "theme": "/rest/api/space/AIRBYTE/theme", "history": "", "homepage": "/rest/api/content/196769"}, "_links": {"webui": "/spaces/AIRBYTE", "self": "https://airbyteio.atlassian.net/wiki/rest/api/space/AIRBYTE"}}, "emitted_at": 1683113211733} {"stream": "group", "data": {"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}, "emitted_at": 1683113212280} diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index 26b38110ddd5d..7c0f38fdd34ad 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -223,6 +223,7 @@ def test_retry_transient_errors(mocker, config, customers, error_cls): def test_cyclic_sieve(caplog): original_logger = logging.getLogger("test") + original_logger.setLevel(logging.DEBUG) sieve = cyclic_sieve(original_logger, fraction=10) for _ in range(20): sieve.info("Ground Control to Major Tom") diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl index c6109ec549df5..09ef5804d0b17 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl @@ -5,8 +5,8 @@ {"stream": "contacts", "data": {"id": 8468604, "title": null, "first_name": "[SAMPLE] Morgan", "last_name": "Minute", "email": "morgan@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748671, "name": "[SAMPLE] Client B"}}, "emitted_at": 1682938831461} {"stream": "contacts", "data": {"id": 8468603, "title": null, "first_name": "[SAMPLE] Sofia", "last_name": "Stopwatch", "email": "sofia@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}}, "emitted_at": 1682938831462} {"stream": "company", "data": {"base_uri": "https://airbyte.harvestapp.com", "full_domain": "airbyte.harvestapp.com", "name": "Airbyte", "is_active": true, "week_start_day": "Monday", "wants_timestamp_timers": false, "time_format": "hours_minutes", "date_format": "%m/%d/%Y", "plan_type": "simple-v4", "expense_feature": true, "invoice_feature": true, "estimate_feature": true, "team_feature": true, "weekly_capacity": 144000, "approval_feature": true, "clock": "12h", "currency": "USD", "currency_code_display": "iso_code_none", "currency_symbol_display": "symbol_before", "decimal_symbol": ".", "thousands_separator": ",", "color_scheme": "orange"}, "emitted_at": 1682938831931} -{"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1682938832494} -{"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1682938832495} +{"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "payment_options": [], "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1686594308269} +{"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "payment_options": [], "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1686594308272} {"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1682938834647} {"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1682938834647} {"stream": "invoice_payments", "data": {"id": 21857618, "amount": 500.0, "paid_at": "2021-05-26T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "", "transaction_id": null, "created_at": "2021-05-26T09:07:06Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": "2021-05-26", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174545}, "emitted_at": 1682938835711} @@ -53,11 +53,11 @@ {"stream": "projects", "data": {"id": 28671451, "name": "Airbyte", "code": null, "is_active": true, "is_billable": true, "is_fixed_fee": false, "bill_by": "none", "budget": null, "budget_by": "none", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": null, "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748673, "name": "Users", "currency": "USD"}}, "emitted_at": 1682938846236} {"stream": "projects", "data": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE", "is_active": true, "is_billable": false, "is_fixed_fee": false, "bill_by": "none", "budget": 160.0, "budget_by": "project", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": "Non-billable projects are perfect for tracking time you don\u2019t want to invoice for. You can use them to track internal projects, vacation/sick time, or pro bono work.", "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}}, "emitted_at": 1682938846236} {"stream": "roles", "data": {"id": 763939, "name": "Sample Role", "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "user_ids": [3758381, 3758382, 3758383, 3758384]}, "emitted_at": 1682938847667} -{"stream": "users", "data": {"id": 3758384, "first_name": "[SAMPLE] Warrin", "last_name": "Wristwatch", "email": "warrin@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": false, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["member"], "permissions_claims": ["expenses:read:own", "expenses:write:own", "timers:read:own", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_4.png"}, "emitted_at": 1682938848246} -{"stream": "users", "data": {"id": 3758383, "first_name": "[SAMPLE] Tamara", "last_name": "Timekeeper", "email": "tamara@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758382, "first_name": "[SAMPLE] Hiromi", "last_name": "Hourglass", "email": "hiromi@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758381, "first_name": "[SAMPLE] Kiran", "last_name": "Kronological", "email": "kiran@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_1.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758380, "first_name": "Airbyte", "last_name": "Developer", "email": "integration-test@airbyte.io", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": null, "cost_rate": null, "roles": [], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://d3s3969qhosaug.cloudfront.net/v2/default-avatars/4144.png"}, "emitted_at": 1682938848247} +{"stream": "users", "data": {"id": 3758384, "first_name": "[SAMPLE] Warrin", "last_name": "Wristwatch", "email": "warrin@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": false, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["member"], "permissions_claims": ["expenses:read:own", "expenses:write:own", "timers:read:own", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_4.png"}, "emitted_at": 1686594810338} +{"stream": "users", "data": {"id": 3758383, "first_name": "[SAMPLE] Tamara", "last_name": "Timekeeper", "email": "tamara@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "saved_reports:write:inactive", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png"}, "emitted_at": 1686594810339} +{"stream": "users", "data": {"id": 3758382, "first_name": "[SAMPLE] Hiromi", "last_name": "Hourglass", "email": "hiromi@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png"}, "emitted_at": 1686594810339} +{"stream": "users", "data": {"id": 3758381, "first_name": "[SAMPLE] Kiran", "last_name": "Kronological", "email": "kiran@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_1.png"}, "emitted_at": 1686594810340} +{"stream": "users", "data": {"id": 3758380, "first_name": "Airbyte", "last_name": "Developer", "email": "integration-test@airbyte.io", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": null, "cost_rate": null, "roles": [], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "saved_reports:write:inactive", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://d3s3969qhosaug.cloudfront.net/v2/default-avatars/4144.png"}, "emitted_at": 1686594810340} {"stream": "billable_rates", "data": {"id": 2164495, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "parent_id": 3758384}, "emitted_at": 1682938850125} {"stream": "billable_rates", "data": {"id": 2164494, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758383}, "emitted_at": 1682938850354} {"stream": "billable_rates", "data": {"id": 2164493, "amount": 125.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758382}, "emitted_at": 1682938850545} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json index c8a20cab052df..0ff2ad3449b8b 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json @@ -162,6 +162,9 @@ } } } + }, + "payment_options": { + "type": ["null", "array"] } } } diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml index 1ffdbd0aac3d7..5cf66816e63f4 100644 --- a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml @@ -29,6 +29,11 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes + empty_streams: + - name: export + bypass_reason: "Data expired too often" + - name: annotations + bypass_reason: "Data expired too often" full_refresh: tests: - config_path: "secrets/config_old.json" diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl index 7daf145e9d95e..d5db2a3f9bb16 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl @@ -1,16 +1,5 @@ -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-29", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037955} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-27", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037956} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-28", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037956} -{"stream": "engage", "data": {"distinct_id": "123@gmail.com", "email": "123@gmail.com", "name": "123", "123": "123456", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042343} -{"stream": "engage", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042345} -{"stream": "engage", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042346} -{"stream": "annotations", "data": {"date": "2023-01-15T12:00:00+01:00", "description": "test", "id": 1138193, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044902} -{"stream": "annotations", "data": {"date": "2023-01-13T12:00:00+01:00", "description": "test123", "id": 1138196, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044904} -{"stream": "annotations", "data": {"date": "2023-01-13T12:00:00+01:00", "description": "test121233", "id": 1138197, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044904} -{"stream": "export", "data": {"event": "Signed up", "import": "True", "insert_id": "29fc2962-6d9c-455d-95ad-95b84f09b9e4", "mp_api_endpoint": "api.mixpanel.com", "mp_api_timestamp_ms": "1685362729000", "distinct_id": "91304156-cafc-4673-a237-623d1129c801", "mp_processing_time_ms": "1685362729276", "time": "2023-05-29T14:17:52Z"}, "emitted_at": 1685362954820} -{"stream": "cohorts", "data": {"id": 1478097, "project_id": 2529987, "name": "Cohort1", "description": "", "data_group_id": null, "count": 2, "is_visible": 1, "created": "2021-09-14 15:57:43"}, "emitted_at": 1684508052373} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1684508059432} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1684508059434} -{"stream": "revenue", "data": {"date": "2023-05-27", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063120} -{"stream": "revenue", "data": {"date": "2023-05-28", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063121} -{"stream": "revenue", "data": {"date": "2023-05-29", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063121} +{"stream": "funnels", "data": {"funnel_id": 8901755, "name": "Onboarding funnel", "date": "2023-06-02", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Viewed Home Page", "goal": "Viewed Home Page", "step_label": "Viewed Home Page", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Signed Up", "goal": "Signed Up", "step_label": "Signed Up", "overall_conv_ratio": 0, "step_conv_ratio": 0}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Onboarding - Action Performed", "goal": "Onboarding - Action Performed", "step_label": "Onboarding - Action Performed", "selector": "(string(properties[\"action\"], \"undefined\") == \"Set a name for your workspace\")", "selector_params": {"step_label": "Onboarding - Action Performed", "bool_op": "and", "property_filter_params_list": [{"filter": {"operator": "==", "operand": ["Set a name for your workspace"]}, "property": {"name": "action", "source": "properties", "type": "string"}, "selected_property_type": "string", "type": "string"}]}, "overall_conv_ratio": 0, "step_conv_ratio": 0}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Onboarding - Action Performed", "goal": "Onboarding - Action Performed", "step_label": "Onboarding - Action Performed", "selector": "(string(properties[\"action\"], \"undefined\") == \"Invite your team to your workspace\")", "selector_params": {"step_label": "Onboarding - Action Performed", "bool_op": "and", "property_filter_params_list": [{"filter": {"operator": "==", "operand": ["Invite your team to your workspace"]}, "property": {"name": "action", "source": "properties", "type": "string"}, "selected_property_type": "string", "type": "string"}]}, "overall_conv_ratio": 0, "step_conv_ratio": 0}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Onboarding - Action Performed", "goal": "Onboarding - Action Performed", "step_label": "Onboarding - Action Performed", "selector": "(string(properties[\"action\"], \"undefined\") == \"Enter the website you want to unblock\")", "selector_params": {"step_label": "Onboarding - Action Performed", "bool_op": "and", "property_filter_params_list": [{"filter": {"operator": "==", "operand": ["Enter the website you want to unblock"]}, "property": {"name": "action", "source": "properties", "type": "string"}, "selected_property_type": "string", "type": "string"}]}, "overall_conv_ratio": 0, "step_conv_ratio": 0}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Onboarding - Action Performed", "goal": "Onboarding - Action Performed", "step_label": "Onboarding - Action Performed", "selector": "(string(properties[\"action\"], \"undefined\") == \"Install Dataline on your website\")", "selector_params": {"step_label": "Onboarding - Action Performed", "bool_op": "and", "property_filter_params_list": [{"filter": {"operator": "==", "operand": ["Install Dataline on your website"]}, "property": {"name": "action", "source": "properties", "type": "string"}, "selected_property_type": "string", "type": "string"}]}, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 6, "worst": 1}}, "emitted_at": 1686595087241} +{"stream": "engage", "data": {"distinct_id": "22885b19-781a-44cd-a8d9-ce46970a3fd6", "browser": "Chrome", "browser_version": "81.0.4044.138", "city": "San Francisco", "country_code": "US", "email": "john+test5@dataline.io", "first_name": "John", "last_name": "Laflur", "name": "John Laflur", "region": "California", "timezone": "America/Los_Angeles", "id": "22885b19-781a-44cd-a8d9-ce46970a3fd6", "last_seen": "2020-05-19T17:53:14"}, "emitted_at": 1686595088617} +{"stream": "cohorts", "data": {"id": 1343181, "project_id": 2117889, "name": "Users in California", "description": "Users in California description", "data_group_id": null, "count": 45, "is_visible": 1, "created": "2021-07-01 22:02:05"}, "emitted_at": 1686595090896} +{"stream": "cohort_members", "data": {"distinct_id": "44b3e80b-894c-480e-9cc0-20cb27784e48", "browser": "Chrome", "browser_version": "85.0.4183.121", "city": "Laguna Niguel", "country_code": "US", "email": "alec@brev.dev", "first_name": "Alec", "last_name": "Fong", "name": "Alec Fong", "region": "California", "timezone": "America/Los_Angeles", "id": "0e2a7bf1-47c5-4c98-b4eb-52859d98adc7", "unblocked": "true", "last_seen": "2020-10-13T22:03:01", "cohort_id": 1343181}, "emitted_at": 1686595093527} +{"stream": "revenue", "data": {"date": "2023-06-02", "amount": 0.0, "count": 121, "paid_count": 0}, "emitted_at": 1686595094576} diff --git a/airbyte-integrations/connectors/source-twitter/README.md b/airbyte-integrations/connectors/source-twitter/README.md index 6ce7dfd8f94a2..59078caa96824 100644 --- a/airbyte-integrations/connectors/source-twitter/README.md +++ b/airbyte-integrations/connectors/source-twitter/README.md @@ -50,7 +50,9 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` To run your integration tests with docker ### Using gradle to run tests diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl index a93cde833349e..74ebcb5e5125f 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl @@ -1,7 +1,4 @@ -{"stream": "forms", "data": {"id": "VWO7mLtl", "type": "quiz", "title": "Connector Extensibility meetup", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "qvDqCNAHuIC8", "ref": "01GHC6KQ5Y0M8VN6XHVAG75J0G", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "default_redirect", "button_text": "Create a typeform"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "ZdzF0rrvsVdB", "title": "What times work for you to visit San Francisco to work with the team?", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "nLpt4rvNjFB3", "ref": "01GHC6KQ5Y155J0F550BGYYS1A", "label": "Dec 12-16"}, {"id": "4xpK9sqA06eL", "ref": "01GHC6KQ5YBATX0CFENVVB5BYG", "label": "Dec 19-23"}, {"id": "jQHb3mqslOsZ", "ref": "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "label": "Jan 9-14"}, {"id": "wS5FKMUnMgqR", "ref": "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "label": "Jan 16-20"}, {"id": "uvmLX80Loava", "ref": "8fffd3a8-1e96-421d-a605-a7029bd55e97", "label": "Jan 22-26"}, {"id": "7ubtgCrW2meb", "ref": "17403cc9-74cd-49d1-856a-be6662b3b497", "label": "Jan30 - Feb3"}, {"id": "51q0g4fTFtYc", "ref": "3a1295b4-97b9-4986-9c37-f1af1d72501d", "label": "Feb 6 - 11"}, {"id": "vi3iwtpETqlb", "ref": "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "label": "Feb 13-17"}, {"id": "iI0hDpta14Kk", "ref": "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee", "label": "Feb 19-24"}]}, "validations": {"required": false}, "type": "multiple_choice", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}], "created_at": "2022-11-08T18:04:03+00:00", "last_updated_at": "2022-11-08T21:10:54+00:00", "published_at": "2022-11-08T21:10:54+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/VWO7mLtl"}}, "emitted_at": 1675773700149} -{"stream": "forms", "data": {"id": "SdMKQYkv", "type": "quiz", "title": "Event Registration (copy)", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/JPnxbU"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": true}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "welcome_screens": [{"id": "3rc53L8DmKJ5", "ref": "70d54ea2e68f27ae", "title": "The annual FormConf is almost here.\n\nWant to come?", "properties": {"show_button": true, "button_text": "Count me in"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/UD82iitWn5XY"}}], "fields": [{"id": "63WvXUnvSCa9", "title": "Great, can we get your full name?", "ref": "ef34b985c51e4131", "properties": {"description": "We'll print this on your event pass."}, "validations": {"required": true}, "type": "short_text"}, {"id": "kwpFrd2lI3ok", "title": "And what's your email address?", "ref": "0c3cabd70157cf16", "properties": {"description": "We'll only use it to send you a confirmation."}, "validations": {"required": true}, "type": "email"}, {"id": "1Ua3d1mzhJwj", "title": "Are you planning on staying for the afterparty?", "ref": "7207397713e2b5e3", "properties": {"description": "We have a surprise guest lined up...", "randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": false, "show_labels": true, "choices": [{"id": "z5hxxjpJl07L", "ref": "bfcc3fbf608583f7", "label": "Yes", "attachment": {"type": "image", "href": "https://images.typeform.com/images/xDriVAzzHfVq"}}, {"id": "37iaitPaS03r", "ref": "5bf390ce5210d38b", "label": "No", "attachment": {"type": "image", "href": "https://images.typeform.com/images/Rn4AmMgzPrYg"}}]}, "validations": {"required": false}, "type": "picture_choice"}, {"id": "MmrPLXSaCF5B", "title": "And do you have any food allergies we should know about?", "ref": "9aaaeeebe70858c4", "properties": {}, "validations": {"required": false}, "type": "short_text"}, {"id": "gurSOcuvNnvb", "title": "Any questions about the event?", "ref": "18842abd9aa9ded4", "properties": {"description": "Write them here and we'll get back to you via email."}, "validations": {"required": false}, "type": "long_text"}, {"id": "fCaCvjCJ57cO", "title": "And finally, would you mind telling us how you heard about the FormConf?", "ref": "261d0775b1f029cb", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": true, "vertical_alignment": false, "choices": [{"id": "IhbdgAo2LHXJ", "ref": "51c76f5fa66c6725", "label": "Social media"}, {"id": "h6Ss8i8gQdOw", "ref": "87408191d53179ee", "label": "Google"}, {"id": "45jv6vFn2nrz", "ref": "8bc14882d0521d6a", "label": "Local advertising"}, {"id": "Iahtxl1jeQwh", "ref": "fce1c86f-fb00-4c33-8085-6e4a4f12ea35", "label": "From a friend"}]}, "validations": {"required": false}, "type": "multiple_choice"}, {"id": "i8rReP3KV7c0", "title": "That's everything. We'll send you an email confirmation with some details a few minutes after you submit this form.\n\nWe hope you're excited as we are :)", "ref": "c6d179ae9c4794e0", "properties": {"button_text": "See you there!", "hide_marks": true}, "type": "statement"}], "created_at": "2021-06-26T14:39:53+00:00", "last_updated_at": "2021-06-27T15:15:56+00:00", "published_at": "2021-06-27T15:15:56+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/SdMKQYkv"}}, "emitted_at": 1675773700338} -{"stream": "forms", "data": {"id": "kRt99jlK", "type": "quiz", "title": "Political Poll [DEMO 2] (copy)", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/wvWlco"}, "settings": {"language": "en", "progress_bar": "percentage", "meta": {"allow_indexing": true}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "welcome_screens": [{"id": "4jguOatzh4QX", "ref": "e8ee8d1500fec6d9", "title": "*National Voting Intentions*", "properties": {"show_button": true, "button_text": "Take Poll", "description": "If you're ok with it, we'd like to know about how you might vote in a general election"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/ty98jF8FppfC"}}], "fields": [{"id": "qthblBc7InVU", "title": "Let's get right to the point:\nIf there was a general election tomorrow, which party would you vote for?", "ref": "8267768033031e53", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "3XbsOhLiGFkv", "ref": "fc3c3f5dbe75a01e", "label": "Center-right party"}, {"id": "FesJYxqJ0SNX", "ref": "61f5e8b36fcdbd91", "label": "Center-left party"}, {"id": "prJzwOH23zsc", "ref": "495e234f7c65d582", "label": "Green party"}, {"id": "6IXNu85c5dOl", "ref": "566c959b6ef92437", "label": "Don't know"}]}, "validations": {"required": true}, "type": "multiple_choice"}, {"id": "rB7FJUThFlu4", "title": "OK, how do you feel about the general direction of our country at the moment?", "ref": "b13b02912db6f287", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": true, "show_labels": true, "choices": [{"id": "Jt8FTorS35Sb", "ref": "b99bb45c5b8c25be", "label": "Going in the wrong direction", "attachment": {"type": "image", "href": "https://images.typeform.com/images/evDEZYspmvjm"}}, {"id": "KUJ0sF2mqG5A", "ref": "2e07534517a152a2", "label": "Going in the right direction", "attachment": {"type": "image", "href": "https://images.typeform.com/images/DgjPyuz9Aphy"}}, {"id": "Dlb3UhA4keI4", "ref": "50be578304fe3e92", "label": "At a standstill", "attachment": {"type": "image", "href": "https://images.typeform.com/images/ZqbqJ6h4zGmM"}}, {"id": "21TAgDkTV2zE", "ref": "d9993c86773807a8", "label": "Unsure", "attachment": {"type": "image", "href": "https://images.typeform.com/images/mNFNeMbxPQMt"}}]}, "validations": {"required": true}, "type": "picture_choice"}, {"id": "vV7ISYSgZ94I", "title": "Thanks, and how do you feel about your own situation this year?", "ref": "f1939629f760be75", "properties": {"start_at_one": true, "steps": 5, "labels": {"left": "Much worse", "center": "About the same", "right": "Much better"}}, "validations": {"required": true}, "type": "opinion_scale"}, {"id": "Mrq4qNeRInni", "title": "Thanks again. Just a couple more questions to go. \nWhich of these issues is most important to you?", "ref": "c52566d91c5052e2", "properties": {"randomize": true, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": false, "show_labels": true, "choices": [{"id": "pTMw2DVbpMyD", "ref": "d9374e70c6cb2f9c", "label": "Taxes & Economy", "attachment": {"type": "image", "href": "https://images.typeform.com/images/6eSzJ9khSfvS"}}, {"id": "dymEoJ3SUDZp", "ref": "d3278566f523cef0", "label": "Labor & Business", "attachment": {"type": "image", "href": "https://images.typeform.com/images/HWfXuXCR3Ls8"}}, {"id": "mOq857nilN8V", "ref": "39ecf093bc5e645b", "label": "Infrastructures", "attachment": {"type": "image", "href": "https://images.typeform.com/images/rW2P45guvd63"}}, {"id": "rnul7dwtsWbg", "ref": "aa1c16d517ffbea0", "label": "Health", "attachment": {"type": "image", "href": "https://images.typeform.com/images/nVsmUESsAzCs"}}, {"id": "Ii1K3mlYioOm", "ref": "4b9a4b0defbf04e5", "label": "Environment", "attachment": {"type": "image", "href": "https://images.typeform.com/images/7ZwHmRi3ZYeg"}}, {"id": "TGZeXHyDgTrQ", "ref": "536d99289a1f80d1", "label": "Education", "attachment": {"type": "image", "href": "https://images.typeform.com/images/Z8qCFjGRD78P"}}, {"id": "1Asa5xFxfuCi", "ref": "68b9b4d37fabf862", "label": "Family & Equality", "attachment": {"type": "image", "href": "https://images.typeform.com/images/YC4Fx6ud6bKq"}}, {"id": "szBe0vqnkCUK", "ref": "3f1692f85c3cd73b", "label": "Military & Defense", "attachment": {"type": "image", "href": "https://images.typeform.com/images/YA746sDt87Xf"}}]}, "validations": {"required": true}, "type": "picture_choice"}, {"id": "aDJXqNTyDXxD", "title": "To finish up, would you mind telling us how you think the current government doing on these issues?", "ref": "f684d8bc4fbca5d3", "properties": {"description": "From 1, doing badly, to 5, doing great...", "button_text": "Continue", "show_button": false, "fields": [{"id": "x9myjwStSn9a", "title": "Economy", "ref": "55c2e5c15f7dccec", "properties": {"shape": "star", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "zaP8jDAArI5x", "title": "National Debt", "ref": "f853e99096a32208", "properties": {"shape": "up", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "VFmcjbHlFTzg", "title": "Employment", "ref": "6f0fd734177ecf27", "properties": {"shape": "user", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "DgGh4ZkRBAyH", "title": "Healthcare", "ref": "b4171ed292dc3cee", "properties": {"shape": "heart", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "yC3UrwN1LKT8", "title": "Education", "ref": "c8dd7d63c26777d9", "properties": {"shape": "pencil", "steps": 5}, "validations": {"required": true}, "type": "rating"}]}, "type": "group"}], "created_at": "2021-06-26T14:39:14+00:00", "last_updated_at": "2021-07-01T10:03:13+00:00", "published_at": "2021-07-01T10:03:13+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/kRt99jlK"}}, "emitted_at": 1675773700751} -{"stream": "forms", "data": {"id": "XtrcGoGJ", "type": "quiz", "title": "Basic Form", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "Xg85PhXqk4HR", "ref": "01F8N53B82QB6T2VM7ASP16198", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "reload", "button_text": "reload"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "8VK4KwNd0DgB", "title": "Hello, what's your name?", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG", "properties": {}, "validations": {"required": false}, "type": "short_text", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}, {"id": "HbNDNK4LLOXB", "title": "Fill your email here:", "ref": "b7980774-f17f-43bc-920b-586e03398f03", "properties": {}, "validations": {"required": false}, "type": "email"}, {"id": "5l7cx4NRa7aX", "title": "enter site", "ref": "e98312ff-df57-4de2-82ae-3617e6dd32ab", "properties": {}, "validations": {"required": false}, "type": "website"}, {"id": "6lGZzhNfrqwB", "title": "Multi-Select question.", "ref": "43153da3-fbbc-443e-b66f-1752770c0e0a", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": false, "vertical_alignment": true, "choices": [{"id": "3HfyxDo5JoXf", "ref": "f83999f6-c869-47cc-af2f-f22b628a0fdb", "label": "choice 3"}, {"id": "03VP9UxCwCLT", "ref": "27b8dfcb-ef16-4ad7-b2be-734ec24c34ca", "label": "choice 4"}, {"id": "ELm7HbFr0OOq", "ref": "ce51ab49-2cce-490d-b831-309337c79fa0", "label": "choice2"}, {"id": "acwDGU8NeO2A", "ref": "74ef0411-0c8a-4c09-a6f3-7a62b0745f68", "label": "choice1"}]}, "validations": {"required": false}, "type": "multiple_choice"}, {"id": "X6dq0mumvtKq", "title": "Nice to meet you, {{field:01F8N53B7KPZ2A1DWGZTTK9SKG}}, how is your day going?", "ref": "01F8N53B8293QHVDDHT84RZR6K", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "vertical_alignment": true, "choices": [{"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}, {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}]}, "validations": {"required": false}, "type": "multiple_choice"}], "created_at": "2021-06-20T16:47:13+00:00", "last_updated_at": "2022-06-20T10:19:58+00:00", "published_at": "2021-06-20T16:47:26+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ"}}, "emitted_at": 1675773700952} +{"stream": "forms", "data": {"id": "VWO7mLtl", "type": "quiz", "title": "Connector Extensibility meetup", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "use_lead_qualification": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "qvDqCNAHuIC8", "ref": "01GHC6KQ5Y0M8VN6XHVAG75J0G", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "default_redirect", "button_text": "Create a typeform"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "ZdzF0rrvsVdB", "title": "What times work for you to visit San Francisco to work with the team?", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "nLpt4rvNjFB3", "ref": "01GHC6KQ5Y155J0F550BGYYS1A", "label": "Dec 12-16"}, {"id": "4xpK9sqA06eL", "ref": "01GHC6KQ5YBATX0CFENVVB5BYG", "label": "Dec 19-23"}, {"id": "jQHb3mqslOsZ", "ref": "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "label": "Jan 9-14"}, {"id": "wS5FKMUnMgqR", "ref": "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "label": "Jan 16-20"}, {"id": "uvmLX80Loava", "ref": "8fffd3a8-1e96-421d-a605-a7029bd55e97", "label": "Jan 22-26"}, {"id": "7ubtgCrW2meb", "ref": "17403cc9-74cd-49d1-856a-be6662b3b497", "label": "Jan30 - Feb3"}, {"id": "51q0g4fTFtYc", "ref": "3a1295b4-97b9-4986-9c37-f1af1d72501d", "label": "Feb 6 - 11"}, {"id": "vi3iwtpETqlb", "ref": "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "label": "Feb 13-17"}, {"id": "iI0hDpta14Kk", "ref": "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee", "label": "Feb 19-24"}]}, "validations": {"required": false}, "type": "multiple_choice", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}], "created_at": "2022-11-08T18:04:03+00:00", "last_updated_at": "2022-11-08T21:10:54+00:00", "published_at": "2022-11-08T21:10:54+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/VWO7mLtl"}}, "emitted_at": 1686590629013} {"stream":"responses","data":{"landing_id":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","token":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","response_id":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","landed_at":"2022-11-08T21:59:53Z","submitted_at":"2022-11-08T22:00:24Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"8a0111039f","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","4xpK9sqA06eL","jQHb3mqslOsZ","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","01GHC6KQ5YBATX0CFENVVB5BYG","1c392fa3-e693-49fe-b334-3a5cddc1db6f","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Dec 19-23","Jan 9-14","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 19-24"]}}]},"emitted_at":1673035160703} {"stream":"responses","data":{"landing_id":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","token":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","response_id":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","landed_at":"2022-11-08T22:08:39Z","submitted_at":"2022-11-08T22:10:04Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"d4b74277d2","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","wS5FKMUnMgqR","jQHb3mqslOsZ","51q0g4fTFtYc","vi3iwtpETqlb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","1c392fa3-e693-49fe-b334-3a5cddc1db6f","3a1295b4-97b9-4986-9c37-f1af1d72501d","54edf52a-c9c7-4bc4-a5a6-bd86115f5adb","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Jan 16-20","Jan 9-14","Feb 6 - 11","Feb 13-17","Feb 19-24"]}}]},"emitted_at":1673035160703} {"stream":"responses","data":{"landing_id":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","token":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","response_id":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","landed_at":"2022-11-09T06:16:08Z","submitted_at":"2022-11-09T06:16:10Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"2be9dd4bab","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","51q0g4fTFtYc","vi3iwtpETqlb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","3a1295b4-97b9-4986-9c37-f1af1d72501d","54edf52a-c9c7-4bc4-a5a6-bd86115f5adb","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 6 - 11","Feb 13-17","Feb 19-24"]}}]},"emitted_at":1673035160855} diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index 982e66615476a..0e8e9a7b93a98 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -13,8 +13,8 @@ {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5138125924367.json", "id": 5138125924367, "assignee_id": null, "group_id": null, "requester_id": 5137812260495, "ticket_id": 123, "score": "offered", "created_at": "2022-07-13T16:02:03Z", "updated_at": "2022-07-13T16:02:03Z", "comment": null}, "emitted_at": 1682939872221} {"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001110696.json", "id": 360001110696, "title": "test police", "description": "for tests", "position": 1, "filter": {"all": [{"field": "assignee_id", "operator": "is", "value": 361089721035}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 61, "business_hours": false}], "created_at": "2021-07-16T11:05:31Z", "updated_at": "2021-07-16T11:05:31Z"}, "emitted_at": 1682939873261} {"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001113715.json", "id": 360001113715, "title": "test police 2", "description": "test police 2", "position": 2, "filter": {"all": [{"field": "organization_id", "operator": "is", "value": 360033549136}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 121, "business_hours": false}], "created_at": "2021-07-16T11:06:01Z", "updated_at": "2021-07-16T11:06:01Z"}, "emitted_at": 1682939873262} -{"stream": "tags", "data": {"name": "test", "count": 2}, "emitted_at": 1682939874989} -{"stream": "tags", "data": {"name": "tag1204", "count": 1}, "emitted_at": 1682939874990} +{"stream": "tags", "data": {"name": "test", "count": 1}, "emitted_at": 1686651830361} +{"stream": "tags", "data": {"name": "tag1204", "count": 1}, "emitted_at": 1686651830362} {"stream": "ticket_audits", "data": {"id": 6764092975119, "ticket_id": 145, "created_at": "2023-04-12T14:04:56Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.34", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 6764092975247, "type": "Change", "value": "low", "field_name": "priority", "previous_value": null}, {"id": 6764092975375, "type": "Change", "value": ["tag1204", "test"], "field_name": "tags", "previous_value": "[]"}, {"id": 6764092975503, "type": "Change", "value": "question", "field_name": "type", "previous_value": null}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1682939882143} {"stream": "ticket_audits", "data": {"id": 5909514817039, "ticket_id": 25, "created_at": "2022-11-22T17:02:04Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 5909514817167, "type": "Notification", "subject": "Request #{{ticket.id}}: How would you rate the support you received?", "body": "Hello {{ticket.requester.name}},\n\nWe'd love to hear what you think of our customer service. Please take a moment to answer one simple question by clicking either link below:\n\n{{satisfaction.rating_section}}\n\nHere's a reminder of what this request was about:\n\n{{ticket.comments_formatted}}\n", "recipients": [360786799676]}, {"id": 5909514817295, "type": "Change", "value": "offered", "field_name": "satisfaction_score", "previous_value": "unoffered"}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Request customer satisfaction rating (system automation)", "id": 360021281435}, "rel": "automation"}}}, "emitted_at": 1682939882145} {"stream": "ticket_audits", "data": {"id": 5909508365711, "ticket_id": 141, "created_at": "2022-11-22T17:02:04Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 5909502407183, "type": "Notification", "subject": "Request #{{ticket.id}}: How would you rate the support you received?", "body": "Hello {{ticket.requester.name}},\n\nWe'd love to hear what you think of our customer service. Please take a moment to answer one simple question by clicking either link below:\n\n{{satisfaction.rating_section}}\n\nHere's a reminder of what this request was about:\n\n{{ticket.comments_formatted}}\n", "recipients": [360786799676]}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Request customer satisfaction rating (system automation)", "id": 360021281435}, "rel": "automation"}}}, "emitted_at": 1682939882146} From 09ec1efcc432190f0b70f7bb9d44b2dfdd335b76 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 13 Jun 2023 18:17:14 +0200 Subject: [PATCH 17/18] connectors-ci: weekly build for alpha connectors (#27300) --- .github/connectors_weekly_build.yml | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/connectors_weekly_build.yml diff --git a/.github/connectors_weekly_build.yml b/.github/connectors_weekly_build.yml new file mode 100644 index 0000000000000..a6048fbc6b06d --- /dev/null +++ b/.github/connectors_weekly_build.yml @@ -0,0 +1,68 @@ +name: Connectors weekly build + +on: + schedule: + # 12PM UTC on Sunday is 2PM CEST, 3PM EEST, 5 PDT. + - cron: "0 12 * * 0" + workflow_dispatch: + inputs: + runs-on: + type: string + default: dev-large-runner + required: true + test-connectors-options: + default: --concurrency=5 --release-stage=alpha + required: true + +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Alpha connectors' }} - on ${{ inputs.runs-on || 'dev-large-runner' }}" + +jobs: + test_connectors: + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Alpha connectors' }} - on ${{ inputs.runs-on || 'dev-large-runner' }}" + timeout-minutes: 8640 # 6 days + runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} + steps: + - name: Get start timestamp + id: get-start-timestamp + run: echo "::set-output name=start-timestamp::$(date +%s)" + - name: Login to DockerHub + run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Extract branch name + shell: bash + if: github.event_name == 'workflow_dispatch' + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Install ci-connector-ops package + run: pip install ./tools/ci_connector_ops\[pipelines]\ + - name: Test connectors + run: | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" + DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then + mkdir -p "$DAGGER_TMP_BINDIR" + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" + fi + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options || '--concurrency=5 --release-stage=alpha' }} test + env: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + CI_REPORT_BUCKET_NAME: "airbyte-ci-reports" + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} + CI_CONTEXT: "nightly_builds" + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} From 6879cdf5d6284fa5b77c8bf9b198287468dd7106 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 13 Jun 2023 18:22:45 +0200 Subject: [PATCH 18/18] weekly build: add missing env var (#27320) --- .github/connectors_weekly_build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/connectors_weekly_build.yml b/.github/connectors_weekly_build.yml index a6048fbc6b06d..2737fc793120a 100644 --- a/.github/connectors_weekly_build.yml +++ b/.github/connectors_weekly_build.yml @@ -66,3 +66,4 @@ jobs: CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} CI_CONTEXT: "nightly_builds" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + CI_JOB_KEY: "weekly_alpha_test"