From 77c7d3e9b363b254983bd9a92ec9c5104e003cfb Mon Sep 17 00:00:00 2001 From: Artur Siepietowski Date: Mon, 7 Nov 2022 21:50:36 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20source:=20Klarna=20(#18385?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding transactions for klarna source integration * Fixing tests * Adding payout stream * Finalizing connector * Adding new lines af EOF * Applying code review suggestions. Adding doc for klarna. Adding title in spec.yaml * run format * format files * add to source def * auto-bump connector version Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 43 ++++ .../connectors/source-klarna/.dockerignore | 6 + .../connectors/source-klarna/Dockerfile | 38 ++++ .../connectors/source-klarna/README.md | 132 ++++++++++++ .../source-klarna/acceptance-test-config.yml | 30 +++ .../source-klarna/acceptance-test-docker.sh | 16 ++ .../connectors/source-klarna/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 26 +++ .../integration_tests/invalid_config.json | 6 + .../integration_tests/sample_config.json | 6 + .../connectors/source-klarna/main.py | 13 ++ .../connectors/source-klarna/requirements.txt | 2 + .../connectors/source-klarna/setup.py | 28 +++ .../source-klarna/source_klarna/__init__.py | 8 + .../source_klarna/schemas/payouts.json | 135 ++++++++++++ .../source_klarna/schemas/transactions.json | 196 ++++++++++++++++++ .../source-klarna/source_klarna/source.py | 118 +++++++++++ .../source-klarna/source_klarna/spec.yaml | 34 +++ .../source-klarna/unit_tests/__init__.py | 3 + .../source-klarna/unit_tests/conftest.py | 23 ++ .../source-klarna/unit_tests/test_source.py | 24 +++ .../source-klarna/unit_tests/test_streams.py | 92 ++++++++ docs/integrations/sources/klarna.md | 61 ++++++ 26 files changed, 1073 insertions(+) create mode 100644 airbyte-integrations/connectors/source-klarna/.dockerignore create mode 100644 airbyte-integrations/connectors/source-klarna/Dockerfile create mode 100644 airbyte-integrations/connectors/source-klarna/README.md create mode 100644 airbyte-integrations/connectors/source-klarna/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-klarna/build.gradle create mode 100644 airbyte-integrations/connectors/source-klarna/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-klarna/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-klarna/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-klarna/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-klarna/main.py create mode 100644 airbyte-integrations/connectors/source-klarna/requirements.txt create mode 100644 airbyte-integrations/connectors/source-klarna/setup.py create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/__init__.py create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/source.py create mode 100644 airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml create mode 100644 airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/klarna.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 57e5bd6856ffb..9a3c063c063b1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -640,6 +640,13 @@ icon: kafka.svg sourceType: database releaseStage: alpha +- name: Klarna + sourceDefinitionId: 60c24725-00ae-490c-991d-55b78c3197e0 + dockerRepository: airbyte/source-klarna + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/klarna + sourceType: api + releaseStage: alpha - name: Klaviyo sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde dockerRepository: airbyte/source-klaviyo diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 82c04af808542..7ab816863d720 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6098,6 +6098,49 @@ supported_destination_sync_modes: [] supported_source_sync_modes: - "append" +- dockerImage: "airbyte/source-klarna:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/klarna" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Klarna Spec" + type: "object" + required: + - "region" + - "playground" + - "username" + - "password" + additionalProperties: true + properties: + region: + title: "Region" + type: "string" + enum: + - "eu" + - "us" + - "oc" + description: "Base url region (For playground eu https://docs.klarna.com/klarna-payments/api/payments-api/#tag/API-URLs).\ + \ Supported 'eu', 'us', 'oc'" + playground: + title: "Playground" + type: "boolean" + description: "Propertie defining if connector is used against playground\ + \ or production environment" + default: false + username: + title: "Username" + type: "string" + description: "Consists of your Merchant ID (eid) - a unique number that\ + \ identifies your e-store, combined with a random string (https://developers.klarna.com/api/#authentication)" + password: + title: "Password" + type: "string" + description: "A string which is associated with your Merchant ID and is\ + \ used to authorize use of Klarna's APIs (https://developers.klarna.com/api/#authentication)" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-klaviyo:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/klaviyo" diff --git a/airbyte-integrations/connectors/source-klarna/.dockerignore b/airbyte-integrations/connectors/source-klarna/.dockerignore new file mode 100644 index 0000000000000..afe3b8fc5abbd --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_klarna +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-klarna/Dockerfile b/airbyte-integrations/connectors/source-klarna/Dockerfile new file mode 100644 index 0000000000000..ce2d2878dd50f --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_klarna ./source_klarna + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-klarna diff --git a/airbyte-integrations/connectors/source-klarna/README.md b/airbyte-integrations/connectors/source-klarna/README.md new file mode 100644 index 0000000000000..7491793a46be3 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/README.md @@ -0,0 +1,132 @@ +# Klarna Source + +This is the repository for the Klarna source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/klarna). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-klarna:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/klarna) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klarna/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source klarna test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-klarna:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-klarna:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-klarna:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klarna:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klarna:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-klarna:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-klarna:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-klarna:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-klarna/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klarna/acceptance-test-config.yml new file mode 100644 index 0000000000000..ee1253808a3d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-klarna:dev +tests: + spec: + - spec_path: "source_klarna/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + # incremental: # TODO Implementation of incremental sync is possible + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-klarna/build.gradle b/airbyte-integrations/connectors/source-klarna/build.gradle new file mode 100644 index 0000000000000..0dfdcdb725864 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_klarna' +} diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/__init__.py b/airbyte-integrations/connectors/source-klarna/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py new file mode 100644 index 0000000000000..950b53b59d416 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-klarna/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..7d03865215b61 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/configured_catalog.json @@ -0,0 +1,26 @@ +{ + "streams": [ + { + "stream": { + "name": "payouts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-klarna/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..7f60f63f406b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "region": "eu", + "playground": true, + "username": "PK57312_4d86a3a57622", + "password": "g9QhHeZtkakKBX26" +} diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-klarna/integration_tests/sample_config.json new file mode 100644 index 0000000000000..3d087e2b34df2 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "region": "eu", + "playground": true, + "username": "", + "password": "" +} diff --git a/airbyte-integrations/connectors/source-klarna/main.py b/airbyte-integrations/connectors/source-klarna/main.py new file mode 100644 index 0000000000000..3d6dca25467cf --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klarna import SourceKlarna + +if __name__ == "__main__": + source = SourceKlarna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klarna/requirements.txt b/airbyte-integrations/connectors/source-klarna/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-klarna/setup.py b/airbyte-integrations/connectors/source-klarna/setup.py new file mode 100644 index 0000000000000..0521ee4647c41 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/setup.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", ""] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "responses~=0.22.0", + "source-acceptance-test", +] + +setup( + name="source_klarna", + description="Source implementation for Klarna.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/__init__.py b/airbyte-integrations/connectors/source-klarna/source_klarna/__init__.py new file mode 100644 index 0000000000000..58ba02ddefa0a --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceKlarna + +__all__ = ["SourceKlarna"] diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json new file mode 100644 index 0000000000000..e600cca16b698 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json @@ -0,0 +1,135 @@ +{ + "type": "object", + "additionalProperties": true, + "required": [ + "totals", + "payment_reference", + "payout_date", + "currency_code", + "merchant_settlement_type", + "merchant_id" + ], + "properties": { + "totals": { + "type": "object", + "additionalProperties": true, + "properties": { + "commission_amount": { + "description": "The total amount of commissions, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "repay_amount": { + "description": "The total amount of money that has been repaid by the merchant from the debt to Klarna, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "sale_amount": { + "description": "The total amount of sales, in minor units", + "example": 500, + "type": "integer", + "format": "int64" + }, + "holdback_amount": { + "description": "The total amount of money withheld by Klarna, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "tax_amount": { + "description": "The total amount of tax, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "settlement_amount": { + "description": "The total amount of the settlement in question, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "fee_correction_amount": { + "description": "The total amount of fee correction, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "reversal_amount": { + "description": "The total amount of reversals, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "release_amount": { + "description": "The total amount of money released from holdback by Klarna, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "return_amount": { + "description": "The total amount of returns, in minor units", + "example": 550, + "type": "integer", + "format": "int64" + }, + "fee_amount": { + "description": "The total amount of fees, in minor units", + "example": 500, + "type": "integer", + "format": "int64" + }, + "charge_amount": { + "description": "The total amount of charges, in minor units. The additional field detailed_type contains the purpose of the charge", + "example": 500, + "type": "integer", + "format": "int64" + }, + "credit_amount": { + "description": "The total amount of credits, in minor units. The additional field detailed_type contains the purpose of the credit", + "example": 500, + "type": "integer", + "format": "int64" + } + } + }, + "payment_reference": { + "description": "The reference id of the payout", + "example": "XISA93DJ", + "type": "string" + }, + "payout_date": { + "description": "ISO 8601 formatted date-time string", + "example": "2016-12-14T07:52:26Z", + "type": "string", + "format": "date-time" + }, + "currency_code": { + "description": "ISO 4217 Currency Code. Like USD, EUR, AUD or GBP.", + "example": "USD", + "type": "string" + }, + "currency_code_of_registration_country": { + "type": "string", + "description": "ISO 4217 Currency Code of the country you are registered in.", + "example": "EUR" + }, + "merchant_settlement_type": { + "description": "Whether the amounts are net or gross", + "example": "NET", + "type": "string", + "enum": ["GROSS", "NET", "GROSS_FEE"] + }, + "merchant_id": { + "description": "The merchant id", + "type": "string" + }, + "transactions": { + "description": "Link to the transactions that are part of this payout", + "example": "https://{settlements_api}/transactions?payment_reference=XISA93DJ", + "type": "string" + } + }, + "$schema": "http://json-schema.org/schema#" +} diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json new file mode 100644 index 0000000000000..e53acfbbe19e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json @@ -0,0 +1,196 @@ +{ + "type": "object", + "additionalProperties": true, + "required": [ + "capture_id", + "sale_date", + "capture_date", + "order_id" + ], + "properties": { + "amount": { + "description": "Total amount of the specific transaction, in minor units", + "example": 2000, + "type": "integer", + "format": "int64" + }, + "capture_id": { + "description": "The Klarna assigned id reference of a specific capture", + "example": "33db6f16-9f43-43fa-a587-cc51411c98e4", + "type": "string" + }, + "merchant_reference1": { + "description": "Merchant assigned reference, typically a reference to an order management system id", + "type": "string" + }, + "sale_date": { + "description": "ISO 8601 formatted date-time string", + "example": "2016-12-14T07:52:26Z", + "type": "string", + "format": "date-time" + }, + "type": { + "description": "The type of transaction.", + "example": "SALE", + "type": "string", + "enum": [ + "COMMISSION", + "SALE", + "REVERSAL", + "RETURN", + "TAX", + "FEE", + "FEE_REFUND", + "CORRECTION", + "REVERSAL_MERCHANT_PROTECTION", + "CHARGE", + "CREDIT", + "HOLDBACK", + "RELEASE" + ] + }, + "capture_date": { + "description": "ISO 8601 formatted date-time string", + "example": "2016-12-14T07:52:26Z", + "type": "string", + "format": "date-time" + }, + "payment_reference": { + "description": "Reference to the specific payout the transaction is part of, if available.", + "example": "XISA93DJ", + "type": "string" + }, + "order_id": { + "description": "The Klarna assigned order id reference", + "example": "ce17b4cb-147f-48b7-b8e6-dde2fa397f04", + "type": "string", + "format": "uuid" + }, + "payout": { + "description": "Link to the payout that this transaction is part of", + "example": "https://{settlements_api}/payouts/XISA93DJ", + "type": "string" + }, + "refund_id": { + "description": "The Klarna assigned id reference of a specific refund", + "example": "ef1baa1f-b42e-44be-b9e4-4b94510b53e5", + "type": "string" + }, + "short_order_id": { + "description": "The Klarna assigned short order id reference", + "example": "shortrid", + "type": "string" + }, + "merchant_reference2": { + "description": "Merchant assigned reference, typically a reference to an order management system id", + "type": "string" + }, + "currency_code": { + "description": "ISO 4217 Currency Code. Like USD, EUR, AUD or GBP.", + "example": "USD", + "type": "string" + }, + "purchase_country": { + "type": "string", + "description": "ISO Alpha-2 Country Code", + "example": "PL" + }, + "vat_rate": { + "type": "integer", + "description": "VAT (Value added tax) rate on Klarna fees", + "example": 2000 + }, + "vat_amount": { + "type": "integer", + "description": "VAT (Value added tax) amount on Klarna fees, in minor units", + "example": 1000 + }, + "shipping_country": { + "type": "string", + "description": "ISO Alpha-2 Country Code", + "example": "PL" + }, + "initial_payment_method_type": { + "type": "string", + "description": "Payment method the consumer chose during checkout", + "example": "direct_debit" + }, + "initial_number_of_installments": { + "type": "integer", + "description": "Number of installments the consumer chose during checkout in case of installment payments", + "example": 3 + }, + "initial_payment_method_monthly_downpayments": { + "type": "integer", + "description": "Number of monthly downpayments that were chosen during the checkout in case of installment payments.", + "example": 12 + }, + "merchant_capture_reference": { + "type": "string", + "description": "Your internal reference to the capture, that has been submitted during capturing an order via API" + }, + "merchant_refund_reference": { + "type": "string", + "description": "Your internal reference to the refund, that has been submitted during refunding an order via API" + }, + "detailed_type": { + "type": "string", + "description": "Detailed description of the transaction type", + "example": "PURCHASE", + "enum": [ + "COMMISSION", + "CREDITED_CORRECTION", + "PURCHASE_FEE_PERCENTAGE", + "PURCHASE_FEE_PERCENTAGE_REFUND", + "LATE_RETURN_FEE", + "PURCHASE_FEE_FIXED", + "EXPIRY_FEE_GROSS", + "EXPIRY_FEE", + "SERVICING_FEE", + "RETURN_FEE", + "EXTRA_INVOICE_FEE", + "PURCHASE_RETURN", + "COMMISSION_RETURN", + "REVERSAL", + "FRAUD_POLICY_CHARGE", + "COMMISSION_RETURN_GROSS", + "FRAUD_POLICY_CREDIT_NET", + "PURCHASE", + "MANUAL_ENTRY", + "LOAN_PAYOUT", + "LOAN_AMORTISATION", + "LOAN_FEE", + "FEE_REFUND", + "PURCHASE_COMMISSION_PERCENTAGE", + "EXTEND_DUE_DATE_FEE", + "TRANSFER_FROM_LEGACY_INTEGRATION", + "FIXED_FEE_CORRECTION_UK", + "PERCENTAGE_FEE_CORRECTION_UK", + "VAT_ON_FEE_CORRECTION_UK", + "FIXED_FEE_CORRECTION_SE", + "PERCENTAGE_FEE_CORRECTION_SE", + "PERCENTAGE_FEE_CORRECTION", + "FIXED_FEE_CORRECTION", + "ROLLING_RESERVE", + "PERCENTAGE_FEES", + "PAYMENT_REMINDER", + "CORRECTION", + "UNDER_REVIEW", + "INSUFFICIENT_BANK_ACCOUNT_DETAILS", + "DISPUTE_FEE", + "DISPUTE_FEE_REFUND" + ] + }, + "tax_in_currency_of_registration_country": { + "type": "integer", + "description": "The tax amount on the respective fee, converted into the currency of your registration country. In case you are a German merchant selling in another currency then EUR or a Swedish merchant selling in another currency then SEK, we convert the VAT amount on the Klarna fees into the currency of the country you are registered in, based on the exchange rate of the capture date.", + "example": 1000 + }, + "currency_code_of_registration_country": { + "type": "string", + "description": "ISO 4217 Currency Code of the country you are registered in.", + "example": "EUR" + } + }, + "$schema": "http://json-schema.org/schema#" +} diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/source.py b/airbyte-integrations/connectors/source-klarna/source_klarna/source.py new file mode 100644 index 0000000000000..0c07cee6b4fc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/source.py @@ -0,0 +1,118 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from urllib.parse import parse_qs, urlparse + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator + + +# Basic full refresh stream +class KlarnaStream(HttpStream, ABC): + def __init__(self, region: str, playground: bool, authenticator: BasicHttpAuthenticator, **kwargs): + self.region = region + self.playground = playground + self.kwargs = kwargs + super().__init__(authenticator=authenticator) + + page_size = 500 + data_api_field: str + + @property + def url_base(self) -> str: + playground_path = "playground." if self.playground else "" + if self.region == "eu": + endpoint = f"https://api.{playground_path}klarna.com/" + else: + endpoint = f"https://api-{self.region}.{playground_path}klarna.com/" + return endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + response_json = response.json() + if "next" in response_json.get("pagination", {}).keys(): + parsed_url = urlparse(response_json["pagination"]["next"]) + query_params = parse_qs(parsed_url.query) + # noinspection PyTypeChecker + return query_params + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + if next_page_token: + return dict(next_page_token) + else: + return {"offset": 0, "size": self.page_size} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + payouts = response.json().get(self.data_api_field, []) + yield from payouts + + +class Payouts(KlarnaStream): + """ + Payouts read from Klarna Settlements API https://developers.klarna.com/api/?json#settlements-api + """ + + primary_key = "payout_date" # TODO verify + data_api_field = "payouts" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/settlements/v1/payouts" + + +class Transactions(KlarnaStream): + """ + Transactions read from Klarna Settlements API https://developers.klarna.com/api/?json#settlements-api + """ + + primary_key = "capture_id" # TODO verify + data_api_field = "transactions" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/settlements/v1/transactions" + + +# Source +class SourceKlarna(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + try: + auth = BasicHttpAuthenticator(username=config["username"], password=config["password"]) + conn_test_stream = Transactions(authenticator=auth, **config) + conn_test_stream.page_size = 1 + conn_test_stream.next_page_token = lambda x: None + records = conn_test_stream.read_records(sync_mode=SyncMode.full_refresh) + # Try to read one value from records iterator + next(records, None) + return True, None + except Exception as e: + print(e) + return False, repr(e) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = BasicHttpAuthenticator(username=config["username"], password=config["password"]) + return [Payouts(authenticator=auth, **config), Transactions(authenticator=auth, **config)] diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml b/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml new file mode 100644 index 0000000000000..e2d7dfc9c71f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml @@ -0,0 +1,34 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/klarna +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Klarna Spec + type: object + required: + - region + - playground + - username + - password + additionalProperties: true + properties: + region: + title: Region + type: string + enum: + - eu + - us + - oc + description: Base url region (For playground eu https://docs.klarna.com/klarna-payments/api/payments-api/#tag/API-URLs). Supported 'eu', 'us', 'oc' + playground: + title: Playground + type: boolean + description: Propertie defining if connector is used against playground or production environment + default: false + username: + title: Username + type: string + description: Consists of your Merchant ID (eid) - a unique number that identifies your e-store, combined with a random string (https://developers.klarna.com/api/#authentication) + password: + title: Password + type: string + description: A string which is associated with your Merchant ID and is used to authorize use of Klarna's APIs (https://developers.klarna.com/api/#authentication) + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py b/airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py b/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py new file mode 100644 index 0000000000000..c1164156a062a --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest as pytest +from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator +from source_klarna import SourceKlarna +from source_klarna.source import KlarnaStream + + +@pytest.fixture(name="source_klarna") +def get_source_klarna(): + return SourceKlarna() + + +@pytest.fixture(name="klarna_config") +def get_klarna_config(): + return dict(playground=False, region="eu", username="user", password="password") + + +@pytest.fixture(name="klarna_stream") +def get_klarna_stream(klarna_config): + return KlarnaStream(authenticator=BasicHttpAuthenticator("", ""), **klarna_config) diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py new file mode 100644 index 0000000000000..79616cc3b02c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import responses +from source_klarna.source import SourceKlarna + + +@responses.activate +def test_check_connection(mocker, source_klarna, klarna_config): + responses.add(responses.GET, "https://api.klarna.com/settlements/v1/transactions?offset=0&size=1", json={}) + + logger_mock, config_mock = MagicMock(), klarna_config + assert source_klarna.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker, klarna_config): + source = SourceKlarna() + config_mock = klarna_config + streams = source.streams(config_mock) + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py new file mode 100644 index 0000000000000..782d85ef6ebf3 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator +from source_klarna.source import KlarnaStream, Payouts, Transactions + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(KlarnaStream, "path", "v0/example_endpoint") + mocker.patch.object(KlarnaStream, "primary_key", "test_primary_key") + mocker.patch.object(KlarnaStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class, klarna_stream): + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"offset": 0, "size": 500} + assert klarna_stream.request_params(**inputs) == expected_params + + +@pytest.mark.parametrize( + "total,count,offset,next_,expected_params", + [ + (9, 4, 0, "https://api.playground.klarna.com/settlements/v1/payouts?offset=4&size=4", {"offset": ["4"], "size": ["4"]}), + (9, 4, 4, "https://api.playground.klarna.com/settlements/v1/payouts?offset=48&size=4", {"offset": ["48"], "size": ["4"]}), + ], +) +def test_next_page_token(patch_base_class, klarna_stream, total, count, offset, next_, expected_params): + response_mock = MagicMock() + response_mock.json.return_value = { + "pagination": { + "total": total, + "count": count, + "offset": offset, + "next": next_, + } + } + inputs = {"response": response_mock} + assert klarna_stream.next_page_token(**inputs) == expected_params + + +@pytest.mark.parametrize( + ("specific_klarna_stream", "response"), + [ + (Payouts, {"payouts": [{}]}), + (Transactions, {"transactions": [{}]}), + ], +) +def test_parse_response(patch_base_class, klarna_config, specific_klarna_stream, response): + mock_response = MagicMock() + mock_response.json.return_value = response + inputs = {"response": mock_response, "stream_state": {}} + stream = specific_klarna_stream(authenticator=BasicHttpAuthenticator("", ""), **klarna_config) + assert next(stream.parse_response(**inputs)) == {} + + +def test_request_headers(patch_base_class, klarna_stream): + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert klarna_stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class, klarna_stream): + expected_method = "GET" + assert klarna_stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry, klarna_stream): + response_mock = MagicMock() + response_mock.status_code = http_status + assert klarna_stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class, klarna_stream): + response_mock = MagicMock() + expected_backoff_time = None + assert klarna_stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/klarna.md b/docs/integrations/sources/klarna.md new file mode 100644 index 0000000000000..a7699f9322f07 --- /dev/null +++ b/docs/integrations/sources/klarna.md @@ -0,0 +1,61 @@ +# Klarna + +This page contains the setup guide and reference information for the Klarna source connector. + +## Prerequisites + +The [Klarna Settlements API](https://developers.klarna.com/api/#settlements-api) is used to get the payouts and transactions for a Klarna account. + +## Setup guide +### Step 1: Set up Klarna + +In order to get an `Username (UID)` and `Password` please go to [this](https://docs.klarna.com/) page here you should find **Merchant Portal** button. Using this button you could log in to your production / playground in proper region. After registration / login you may find and create `Username (UID)` and `Password` in settings tab. + +:::note + +Klarna Source Connector does not support OAuth at this time due to limitations outside of control. + +::: + +## Step 2: Set up the Klarna connector in Airbyte + +### For Airbyte Open Source: +1. Navigate to the Airbyte Open Source dashboard +2. Set the name for your source +3. Choose if your account is sandbox +4. Enter your username +5. Enter your password +6. Enter the date you want your sync to start from +7. Click **Set up source** + +## Supported sync modes + +The Klarna source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :------------------------ |:-----------| +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | + + +## Supported Streams + +This Source is capable of syncing the following Klarna Settlements Streams: + +* [Payouts](https://developers.klarna.com/api/#settlements-api-get-all-payouts) +* [Transactions](https://developers.klarna.com/api/#settlements-api-get-transactions) + +## Performance considerations + +Klarna API has [rate limiting](https://developers.klarna.com/api/#api-rate-limiting) + +**Production environments**: the API rate limit is 20 create-sessions per second on average measured over a 1-minute period. For the other operations, the API limit is 200 requests per second on average, measured over a 1 minute period +**Playground environments**: the API rate limit is one quarter (1/4th) of the rate limits of production environments. + +Connector will handle an issue with rate limiting as Klarna returns 429 status code when limits are reached + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------| +| 0.1.0 | 2022-10-24 | [18385](https://github.com/airbytehq/airbyte/pull/18385) | Klarna Settlements Payout and Transactions API |