From cc43bd7405d8e4aef543e0c77f87ada1322d76c5 Mon Sep 17 00:00:00 2001 From: Mikhail Masyagin Date: Thu, 17 Aug 2023 20:33:23 +0300 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8Source=20Plaid:=20Migrate=20Python=20C?= =?UTF-8?q?DK=20to=20No-code=20CDK=20(#29127)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sajarin Co-authored-by: sh4sh <6833405+sh4sh@users.noreply.github.com> --- .../connectors/source-plaid/Dockerfile | 2 +- .../source-plaid/acceptance-test-config.yml | 37 +- .../source-plaid/acceptance-test-docker.sh | 1 + .../configured_catalog_incremental.json | 107 +++++ .../connectors/source-plaid/metadata.yaml | 2 +- .../connectors/source-plaid/requirements.txt | 2 +- .../connectors/source-plaid/setup.py | 11 +- .../source-plaid/source_plaid/__init__.py | 2 +- .../source-plaid/source_plaid/manifest.yaml | 374 ++++++++++++++++++ .../source_plaid/schemas/balance.json | 13 - .../source_plaid/schemas/transaction.json | 63 --- .../source-plaid/source_plaid/source.py | 148 +------ .../source-plaid/source_plaid/spec.json | 40 -- docs/integrations/sources/plaid.md | 1 + 14 files changed, 526 insertions(+), 277 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json create mode 100644 airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json delete mode 100644 airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json delete mode 100644 airbyte-integrations/connectors/source-plaid/source_plaid/spec.json diff --git a/airbyte-integrations/connectors/source-plaid/Dockerfile b/airbyte-integrations/connectors/source-plaid/Dockerfile index dca1963cb8e23..5d29cf2675505 100644 --- a/airbyte-integrations/connectors/source-plaid/Dockerfile +++ b/airbyte-integrations/connectors/source-plaid/Dockerfile @@ -34,5 +34,5 @@ COPY source_plaid ./source_plaid ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-plaid diff --git a/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml index ba1479d67b7ba..6375b20c3b4af 100644 --- a/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml @@ -1,24 +1,31 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-plaid:dev -tests: +acceptance_tests: spec: - - spec_path: "source_plaid/spec.json" + tests: + - spec_path: "source_plaid/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a78..b6d65deeccb43 --- a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json new file mode 100644 index 0000000000000..2effb0c1b0898 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,107 @@ +{ + "streams": [ + { + "stream": { + "name": "balance", + "supported_sync_modes": ["full_refresh"], + "json_schema": { + "required": ["account_id", "current"], + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "available": { + "type": ["number", "null"] + }, + "current": { + "type": "number" + }, + "iso_currency_code": { + "type": ["string", "null"] + }, + "limit": { + "type": ["number", "null"] + }, + "unofficial_currency_code": { + "type": ["string", "null"] + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transaction", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": "object", + "required": [ + "account_id", + "amount", + "iso_currency_code", + "name", + "transaction_id", + "category", + "date", + "transaction_type" + ], + "properties": { + "account_id": { "type": "string" }, + "amount": { "type": "number" }, + "category": { "type": "array", "items": { "type": "string" } }, + "category_id": { "type": ["string", "null"] }, + "date": { "type": "string" }, + "iso_currency_code": { "type": "string" }, + "name": { "type": "string" }, + "payment_channel": { "type": ["string", "null"] }, + "pending": { "type": ["boolean", "null"] }, + "transaction_id": { "type": "string" }, + "transaction_type": { "type": "string" }, + "location": { + "type": ["object", "null"], + "properties": { + "address": { "type": ["string", "null"] }, + "city": { "type": ["string", "null"] }, + "country": { "type": ["string", "null"] }, + "lat": { "type": ["string", "null"] }, + "lon": { "type": ["string", "null"] }, + "postal_code": { "type": ["string", "null"] }, + "region": { "type": ["string", "null"] }, + "store_number": { "type": ["string", "null"] } + } + }, + "payment_meta": { + "type": ["object", "null"], + "properties": { + "by_order_of": { "type": ["string", "null"] }, + "payee": { "type": ["string", "null"] }, + "payer": { "type": ["string", "null"] }, + "payment_method": { "type": ["string", "null"] }, + "payment_processor": { "type": ["string", "null"] }, + "ppd_id": { "type": ["string", "null"] }, + "reason": { "type": ["string", "null"] }, + "reference_number": { "type": ["string", "null"] } + } + }, + "account_owner": { "type": ["string", "null"] }, + "authorized_date": { "type": ["string", "null"] }, + "authorized_datetime": { "type": ["string", "null"] }, + "check_number": { "type": ["string", "null"] }, + "datetime": { "type": ["string", "null"] }, + "merchant_name": { "type": ["string", "null"] }, + "pending_transaction_id": { "type": ["string", "null"] }, + "personal_finance_category": { "type": ["string", "null"] }, + "transaction_code": { "type": ["string", "null"] }, + "unofficial_currency_code": { "type": ["string", "null"] } + } + } + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-plaid/metadata.yaml b/airbyte-integrations/connectors/source-plaid/metadata.yaml index 3e95b53abb961..e05ecb3ad445c 100644 --- a/airbyte-integrations/connectors/source-plaid/metadata.yaml +++ b/airbyte-integrations/connectors/source-plaid/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: ed799e2b-2158-4c66-8da4-b40fe63bc72a - dockerImageTag: 0.3.2 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-plaid githubIssueLabel: source-plaid icon: plaid.svg diff --git a/airbyte-integrations/connectors/source-plaid/requirements.txt b/airbyte-integrations/connectors/source-plaid/requirements.txt index 7b9114ed5867e..5b8864c417d30 100644 --- a/airbyte-integrations/connectors/source-plaid/requirements.txt +++ b/airbyte-integrations/connectors/source-plaid/requirements.txt @@ -1,2 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e . +-e ../../bases/connector-acceptance-test diff --git a/airbyte-integrations/connectors/source-plaid/setup.py b/airbyte-integrations/connectors/source-plaid/setup.py index 820c6bb4cb47d..bcbf0b34a80c4 100644 --- a/airbyte-integrations/connectors/source-plaid/setup.py +++ b/airbyte-integrations/connectors/source-plaid/setup.py @@ -5,12 +5,15 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "plaid-python"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "pytest~=6.1", + "requests-mock~=1.9.3", + "connector-acceptance-test", ] setup( @@ -20,7 +23,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py b/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py index 8ff627c962f9b..2891fc126707b 100644 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml b/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml new file mode 100644 index 0000000000000..a2977194f7047 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml @@ -0,0 +1,374 @@ +version: 0.50.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - balance +streams: + - type: DeclarativeStream + name: balance + primary_key: + - account_id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + account_id: + type: string + available: + type: + - 'null' + - number + current: + type: number + iso_currency_code: + type: + - 'null' + - string + limit: + type: + - 'null' + - number + unofficial_currency_code: + type: + - 'null' + - string + required: + - account_id + - current + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{config['plaid_env']}}.plaid.com + path: /accounts/balance/get + http_method: POST + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: + secret: '{{config[''api_key'']}}' + options: + min_last_updated_datetime: '{{format_datetime(config[''start_date''], ''%Y-%m-%dT%H:%M:%SZ'')}}' + client_id: '{{config[''client_id'']}}' + access_token: '{{config[''access_token'']}}' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - accounts + - '*' + paginator: + type: NoPagination + transformations: + - type: AddFields + fields: + - path: + - available + value: '{{record[''balances''][''available'']}}' + - type: AddFields + fields: + - path: + - current + value: '{{record[''balances''][''current'']}}' + - type: AddFields + fields: + - path: + - iso_currency_code + value: '{{record[''balances''][''iso_currency_code'']}}' + - type: AddFields + fields: + - path: + - limit + value: '{{record[''balances''][''limit'']}}' + - type: AddFields + fields: + - path: + - unofficial_currency_code + value: '{{record[''balances''][''unofficial_currency_code'']}}' + - type: RemoveFields + field_pointers: + - - balances + - type: RemoveFields + field_pointers: + - - mask + - type: RemoveFields + field_pointers: + - - name + - type: RemoveFields + field_pointers: + - - official_name + - type: RemoveFields + field_pointers: + - - subtype + - type: RemoveFields + field_pointers: + - - type + - type: DeclarativeStream + name: transaction + primary_key: + - transaction_id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + account_id: + type: string + account_owner: + type: + - string + - 'null' + amount: + type: number + authorized_date: + type: + - string + - 'null' + authorized_datetime: + type: + - string + - 'null' + category: + items: + type: string + type: array + category_id: + type: + - string + - 'null' + check_number: + type: + - string + - 'null' + date: + type: string + datetime: + type: + - string + - 'null' + iso_currency_code: + type: string + location: + properties: + address: + type: + - string + - 'null' + city: + type: + - string + - 'null' + country: + type: + - string + - 'null' + lat: + type: + - string + - 'null' + lon: + type: + - string + - 'null' + postal_code: + type: + - string + - 'null' + region: + type: + - string + - 'null' + store_number: + type: + - string + - 'null' + type: + - object + - 'null' + merchant_name: + type: + - string + - 'null' + name: + type: string + payment_channel: + type: + - string + - 'null' + payment_meta: + properties: + by_order_of: + type: + - string + - 'null' + payee: + type: + - string + - 'null' + payer: + type: + - string + - 'null' + payment_method: + type: + - string + - 'null' + payment_processor: + type: + - string + - 'null' + ppd_id: + type: + - string + - 'null' + reason: + type: + - string + - 'null' + reference_number: + type: + - string + - 'null' + type: + - object + - 'null' + pending: + type: + - boolean + - 'null' + pending_transaction_id: + type: + - string + - 'null' + personal_finance_category: + type: + - string + - 'null' + transaction_code: + type: + - string + - 'null' + transaction_id: + type: string + transaction_type: + type: string + unofficial_currency_code: + type: + - string + - 'null' + required: + - account_id + - amount + - iso_currency_code + - name + - transaction_id + - category + - date + - transaction_type + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{config['plaid_env']}}.plaid.com + path: /transactions/get + http_method: POST + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: + secret: '{{config[''api_key'']}}' + options: + offset: '{{ next_page_token[''next_page_token''] }}' + client_id: '{{config[''client_id'']}}' + access_token: '{{config[''access_token'']}}' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - transactions + - '*' + paginator: + type: DefaultPaginator + pagination_strategy: + type: OffsetIncrement + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date + datetime_format: '%Y-%m-%d' + start_time_option: + type: RequestOption + field_name: start_date + inject_into: body_json + end_time_option: + type: RequestOption + field_name: end_date + inject_into: body_json + start_datetime: + type: MinMaxDatetime + datetime: '{{config[''start_date'']}}' + datetime_format: '%Y-%m-%d' + end_datetime: + type: MinMaxDatetime + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' +spec: + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - access_token + - api_key + - client_id + - plaid_env + properties: + access_token: + type: string + order: 0 + title: Access Token + description: The end-user's Link access token. + airbyte_secret: true + api_key: + type: string + order: 1 + title: API Key + description: The Plaid API key to use to hit the API. + airbyte_secret: true + client_id: + type: string + order: 2 + title: Client ID + description: The Plaid client id. + plaid_env: + enum: + - sandbox + - development + - production + type: string + order: 3 + title: Plaid Environment + description: The Plaid environment. + start_date: + type: string + order: 4 + title: Start Date + format: date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + description: >- + The date from which you'd like to replicate data for Plaid in the + format YYYY-MM-DD. All data generated after this date will be + replicated. + additionalProperties: true + documentation_url: https://example.org + type: Spec +metadata: + autoImportSchema: + balance: false + transaction: false diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json b/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json deleted file mode 100644 index e0b4885ca3bce..0000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["account_id", "current"], - "properties": { - "account_id": { "type": "string" }, - "available": { "type": ["number", "null"] }, - "current": { "type": "number" }, - "iso_currency_code": { "type": ["string", "null"] }, - "limit": { "type": ["number", "null"] }, - "unofficial_currency_code": { "type": ["string", "null"] } - } -} diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json b/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json deleted file mode 100644 index dba0a472d568c..0000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "account_id", - "amount", - "iso_currency_code", - "name", - "transaction_id", - "category", - "date", - "transaction_type" - ], - "properties": { - "account_id": { "type": "string" }, - "amount": { "type": "number" }, - "category": { "type": "array", "items": { "type": "string" } }, - "category_id": { "type": ["string", "null"] }, - "date": { "type": "string" }, - "iso_currency_code": { "type": "string" }, - "name": { "type": "string" }, - "payment_channel": { "type": ["string", "null"] }, - "pending": { "type": ["boolean", "null"] }, - "transaction_id": { "type": "string" }, - "transaction_type": { "type": "string" }, - "location": { - "type": ["object", "null"], - "properties": { - "address": { "type": ["string", "null"] }, - "city": { "type": ["string", "null"] }, - "country": { "type": ["string", "null"] }, - "lat": { "type": ["string", "null"] }, - "lon": { "type": ["string", "null"] }, - "postal_code": { "type": ["string", "null"] }, - "region": { "type": ["string", "null"] }, - "store_number": { "type": ["string", "null"] } - } - }, - "payment_meta": { - "type": ["object", "null"], - "properties": { - "by_order_of": { "type": ["string", "null"] }, - "payee": { "type": ["string", "null"] }, - "payer": { "type": ["string", "null"] }, - "payment_method": { "type": ["string", "null"] }, - "payment_processor": { "type": ["string", "null"] }, - "ppd_id": { "type": ["string", "null"] }, - "reason": { "type": ["string", "null"] }, - "reference_number": { "type": ["string", "null"] } - } - }, - "account_owner": { "type": ["string", "null"] }, - "authorized_date": { "type": ["string", "null"] }, - "authorized_datetime": { "type": ["string", "null"] }, - "check_number": { "type": ["string", "null"] }, - "datetime": { "type": ["string", "null"] }, - "merchant_name": { "type": ["string", "null"] }, - "pending_transaction_id": { "type": ["string", "null"] }, - "personal_finance_category": { "type": ["string", "null"] }, - "transaction_code": { "type": ["string", "null"] }, - "unofficial_currency_code": { "type": ["string", "null"] } - } -} diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/source.py b/airbyte-integrations/connectors/source-plaid/source_plaid/source.py index 29961fd57c0a2..6da533a6f6e09 100644 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/source.py +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/source.py @@ -2,145 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import json -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import plaid -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from plaid.api import plaid_api -from plaid.model.accounts_balance_get_request import AccountsBalanceGetRequest -from plaid.model.accounts_balance_get_request_options import AccountsBalanceGetRequestOptions -from plaid.model.transactions_get_request import TransactionsGetRequest -from plaid.model.transactions_get_request_options import TransactionsGetRequestOptions +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -SPEC_ENV_TO_PLAID_ENV = { - "production": plaid.Environment.Production, - "development": plaid.Environment.Development, - "sandbox": plaid.Environment.Sandbox, -} +WARNING: Do not modify this file. +""" -class PlaidStream(Stream): - def __init__(self, config: Mapping[str, Any]): - plaid_config = plaid.Configuration( - host=SPEC_ENV_TO_PLAID_ENV[config["plaid_env"]], api_key={"clientId": config["client_id"], "secret": config["api_key"]} - ) - api_client = plaid.ApiClient(plaid_config) - self.client = plaid_api.PlaidApi(api_client) - self.access_token = config["access_token"] - self.start_date = datetime.datetime.strptime(config.get("start_date"), "%Y-%m-%d").date() if config.get("start_date") else None - - -class BalanceStream(PlaidStream): - @property - def name(self): - return "balance" - - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return "account_id" - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - min_last_updated_datetime = datetime.datetime.strptime( - datetime.datetime.strftime(self.start_date, "%y-%m-%dT%H:%M:%SZ"), - "%y-%m-%dT%H:%M:%S%z", - ) - options = AccountsBalanceGetRequestOptions(min_last_updated_datetime=min_last_updated_datetime) - getRequest = AccountsBalanceGetRequest(access_token=self.access_token, options=options) - balance_response = self.client.accounts_balance_get(getRequest) - for balance in balance_response["accounts"]: - message_dict = balance["balances"].to_dict() - message_dict["account_id"] = balance["account_id"] - yield message_dict - - -class IncrementalTransactionStream(PlaidStream): - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return "transaction_id" - - @property - def name(self): - return "transaction" - - @property - def source_defined_cursor(self) -> bool: - return True - - @property - def cursor_field(self) -> Union[str, List[str]]: - return "date" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - return {"date": latest_record.get("date")} - - def _get_transactions_response(self, start_date, end_date=datetime.datetime.utcnow().date(), offset=0): - options = TransactionsGetRequestOptions() - options.offset = offset - - return self.client.transactions_get( - TransactionsGetRequest(access_token=self.access_token, start_date=start_date, end_date=end_date, options=options) - ) - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - date = stream_state.get("date") - all_transactions = [] - - if not date: - date = datetime.date.fromtimestamp(0) - else: - date = datetime.date.fromisoformat(date) - if date >= datetime.datetime.utcnow().date(): - return - - if self.start_date: - date = max(self.start_date, date) - - response = self._get_transactions_response(date) - all_transactions.extend(response.transactions) - num_total_transactions = response.total_transactions - - while len(all_transactions) < num_total_transactions: - response = self._get_transactions_response(date, offset=len(all_transactions)) - all_transactions.extend(response.transactions) - - yield from map(lambda x: x.to_dict(), sorted(all_transactions, key=lambda t: t["date"])) - - -class SourcePlaid(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - try: - plaid_config = plaid.Configuration( - host=SPEC_ENV_TO_PLAID_ENV[config["plaid_env"]], api_key={"clientId": config["client_id"], "secret": config["api_key"]} - ) - api_client = plaid.ApiClient(plaid_config) - client = plaid_api.PlaidApi(api_client) - try: - request = AccountsBalanceGetRequest(access_token=config["access_token"]) - client.accounts_balance_get(request) - return True, None - except plaid.ApiException as e: - response = json.loads(e.body) - return False, response - except Exception as error: - return False, error - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [BalanceStream(config), IncrementalTransactionStream(config)] +# Declarative Source +class SourcePlaid(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json b/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json deleted file mode 100644 index 09605304314fc..0000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "documentationUrl": "https://plaid.com/docs/api/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["access_token", "api_key", "client_id", "plaid_env"], - "additionalProperties": true, - "properties": { - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The end-user's Link access token." - }, - "api_key": { - "title": "API Key", - "type": "string", - "description": "The Plaid API key to use to hit the API.", - "airbyte_secret": true - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Plaid client id" - }, - "plaid_env": { - "title": "Plaid Environment", - "type": "string", - "enum": ["sandbox", "development", "production"], - "description": "The Plaid environment" - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Plaid in the format YYYY-MM-DD. All data generated after this date will be replicated.", - "examples": ["2021-03-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - } -} diff --git a/docs/integrations/sources/plaid.md b/docs/integrations/sources/plaid.md index 889ffb39ea843..5caaffda979db 100644 --- a/docs/integrations/sources/plaid.md +++ b/docs/integrations/sources/plaid.md @@ -70,6 +70,7 @@ This guide will walk through how to create the credentials you need to run this | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------ | +| 0.4.0 | 2023-08-17 | [29127](https://github.com/airbytehq/airbyte/pull/29127) | Rewrote connector to no-code SDK | | 0.3.2 | 2022-08-02 | [15231](https://github.com/airbytehq/airbyte/pull/15231) | Added min_last_updated_datetime support for Capital One items | | 0.3.1 | 2022-03-31 | [11104](https://github.com/airbytehq/airbyte/pull/11104) | Fix 100 record limit and added start_date | | 0.3.0 | 2022-01-05 | [7977](https://github.com/airbytehq/airbyte/pull/7977) | Migrate to Python CDK + add transaction stream |